diff --git a/.swiftpm/xcode/xcshareddata/xcschemes/GrAIdient.xcscheme b/.swiftpm/xcode/xcshareddata/xcschemes/GrAIdient.xcscheme index 89608c19..01e205ca 100644 --- a/.swiftpm/xcode/xcshareddata/xcschemes/GrAIdient.xcscheme +++ b/.swiftpm/xcode/xcshareddata/xcschemes/GrAIdient.xcscheme @@ -114,7 +114,7 @@ # -Jean-François Reboud +Peden Aurélien +Reboud Jean-François diff --git a/CHANGELOG.md b/CHANGELOG.md index 26862b72..c6f5db04 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,63 @@ All notable changes to this project will be documented in this file. ## [unreleased] +## 0.3.0 (2023-08-04) + +### Features + +🪜 **feat:** BCE1D, BCE2D, VQ2D & VQSeq as losses ([#101](https://github.com/owkin/GrAIdient/pull/101))\ +🪜 **layer_seq:** VQSeq ([#100](https://github.com/owkin/GrAIdient/pull/100))\ +🪜 **layer_2d:** loosen range contraint in ColorJitterHSV ([#98](https://github.com/owkin/GrAIdient/pull/98))\ +🪜 **layer_2d:** SimilarityError2D & dirty losses ([#97](https://github.com/owkin/GrAIdient/pull/97))\ +🪜 **layer_2d:** ColorJitterHSV, Image & ImageTests ([#93](https://github.com/owkin/GrAIdient/pull/93))\ +🪜 **layer_2d:** Flip2D & config_kernels ([#92](https://github.com/owkin/GrAIdient/pull/92))\ +🪜 **layer_2d:** SimilarityBatchError2D ([#88](https://github.com/owkin/GrAIdient/pull/88))\ +🪜 **layer_2d:** Normalize2D ([#87](https://github.com/owkin/GrAIdient/pull/87))\ +🪜 **layer_2d:** SelfCorrelate2D ([#86](https://github.com/owkin/GrAIdient/pull/86))\ +🪜 **layer_2d**: VQ2D ([#81](https://github.com/owkin/GrAIdient/pull/81))\ +🪜 **layer_seq**: Adding new layer SelectNeuronsSeq ([#77](https://github.com/owkin/GrAIdient/pull/77))\ +⚙️ **core:** GELU activation function ([#73](https://github.com/owkin/GrAIdient/pull/73))\ +🪜 **layer_seq:** ValueSeq ([#69](https://github.com/owkin/GrAIdient/pull/69))\ +🪜 **layer_seq:** SoftmaxSeq ([#68](https://github.com/owkin/GrAIdient/pull/68))\ +🪜 **layer_seq:** QuerySeq ([#67](https://github.com/owkin/GrAIdient/pull/67))\ +🪜 **layer_seq:** LayerNormSeq & LayerNormalization ([#66](https://github.com/owkin/GrAIdient/pull/66))\ +🪜 **layer_seq:** FullyConnectedSeq ([#65](https://github.com/owkin/GrAIdient/pull/65))\ +🪜 **layer_seq:** Constant12Seq & Constant2Seq ([#64](https://github.com/owkin/GrAIdient/pull/64))\ +🪜 **layer_seq:** Concat1Seq & Concat2Seq ([#63](https://github.com/owkin/GrAIdient/pull/63))\ +🪜 **layer_seq:** SumSeq ([#62](https://github.com/owkin/GrAIdient/pull/62))\ +🪜 **layer_2d:** MSE2D & LayerOutput2D ([#61](https://github.com/owkin/GrAIdient/pull/61))\ +🪜 **layer_seq:** FullyConnectedPatch & base classes ([#60](https://github.com/owkin/GrAIdient/pull/60))\ +🪜 **layer_2d:** Constant2D ([#56](https://github.com/owkin/GrAIdient/pull/56))\ +🪜 **layer_2d:** AdaIN ([#55](https://github.com/owkin/GrAIdient/pull/55))\ +🪜 **layer_2d:** InstanceNorm2D & InstanceNormalization ([#54](https://github.com/owkin/GrAIdient/pull/54)) + +### Bug Fixes + +🐛 **layer_2d**: align Convolution & Deconvolution on PyTorch ([#84](https://github.com/owkin/GrAIdient/pull/84))\ +🐛 **fix**: numerical stability of tanh for GELU ([#83](https://github.com/owkin/GrAIdient/pull/83))\ +🐛 **fix:** numerical instability of Softmax ([#76](https://github.com/owkin/GrAIdient/pull/76))\ +🐛 **fix:** update ValueSeq operation ([#72](https://github.com/owkin/GrAIdient/pull/72)) + +### Miscellaneous Tasks + +🔨 **refactor:** throwable init ([#103](https://github.com/owkin/GrAIdient/pull/103))\ +🔨 **refactor:** dims checks for inputs and outputs ([#102](https://github.com/owkin/GrAIdient/pull/102))\ +🔨 **layer_2d:** expose indices in VQ2D ([#99](https://github.com/owkin/GrAIdient/pull/99))\ +🔨 **core:** LayerWeightInit ([#96](https://github.com/owkin/GrAIdient/pull/96))\ +🚨 **test**: FlowAccumulateTrainer ([#95](https://github.com/owkin/GrAIdient/pull/95))\ +🚨 **examples**: compare training with PyTorch ([#94](https://github.com/owkin/GrAIdient/pull/94))\ +🔨 **layer_2d:** remove computeVQ ([#91](https://github.com/owkin/GrAIdient/pull/91))\ +🔨 **layer_2d:** API for random transforms ([#90](https://github.com/owkin/GrAIdient/pull/90))\ +🚀 **perf:** enhance Normalize122D with reduce ([#89](https://github.com/owkin/GrAIdient/pull/89))\ +🚨 **integration**: resize alignment with PyTorch ([#85](https://github.com/owkin/GrAIdient/pull/85))\ +🔨 **layer_seq**: SelectSeq ([#82](https://github.com/owkin/GrAIdient/pull/82))\ +🚀 **examples**: AutoEncoder models ([#79](https://github.com/owkin/GrAIdient/pull/79))\ +🚀 **layer_seq**: factorize by nbHeads ([#78](https://github.com/owkin/GrAIdient/pull/78))\ +🚀 **examples:** make Transformer example very simple ([#75](https://github.com/owkin/GrAIdient/pull/75))\ +🚀 **examples:** adding Transformer training example ([#74](https://github.com/owkin/GrAIdient/pull/74))\ +🚨 **integration:** update & validate LayerNormSeq ([#71](https://github.com/owkin/GrAIdient/pull/71))\ +🚨 **integration:** validate MultiHeadAttention & fix Softmax stability ([#70](https://github.com/owkin/GrAIdient/pull/70)) + ## 0.2.0 (2023-02-27) ### Features @@ -54,7 +111,7 @@ All notable changes to this project will be documented in this file. 🔨 **refactor:** remove transaction ([#31](https://github.com/owkin/GrAIdient/pull/31))\ 🚨 **integration:** activate DecorrelateRGB in test ([#29](https://github.com/owkin/GrAIdient/pull/29))\ 🚨 **integration:** test IDFT and complex numbers ([#28](https://github.com/owkin/GrAIdient/pull/28))\ -🔨 **tests:** factorize transform tests ([#26](https://github.com/owkin/GrAIdient/pull/26))\ +🔨 **test:** factorize transform tests ([#26](https://github.com/owkin/GrAIdient/pull/26))\ 👷 **ci:** remove swift action ([#20](https://github.com/owkin/GrAIdient/pull/20))\ 👷 **ci:** remove LFS ([#17](https://github.com/owkin/GrAIdient/pull/17)) diff --git a/Docs/Architecture/GrAITests.md b/Docs/Architecture/GrAITests.md index 4b5fa53f..5ce128dc 100644 --- a/Docs/Architecture/GrAITests.md +++ b/Docs/Architecture/GrAITests.md @@ -18,6 +18,9 @@ that every layer, optimizer, activation function ... is tested. the execution context (the model CPU will be executed on the GPU and vice versa) +- accumulate tests: compare gradients computed in CPU and GPU + after accumulating them + - inference tests: compare loss in CPU and GPU during the inference phase - load tests: compare loss in CPU and GPU after loading models from the disk diff --git a/Docs/Concepts/MODEL.md b/Docs/Concepts/MODEL.md index 18a55f74..8252ea60 100644 --- a/Docs/Concepts/MODEL.md +++ b/Docs/Concepts/MODEL.md @@ -196,16 +196,29 @@ cnn.weights = myCNNWeights classifier.weights = myClassifierWeights ``` +### Generate Model's Weights + +It is also possible not to set the `weights` at all and have them +generated by the `Model` thanks to its `weightInitClass` API. +The following initialization schemes are available for the moment: + +- Xavier uniform +- Xavier normal +- Kaiming uniform +- Kaiming normal + +By default, the Xavier uniform initialization scheme is used. + ### Model Loaded from the Disk -When a model has been loaded from the disk -(see [previous paragraph](#initialize-links)), there is no need to use -the `weights` API: the cache for the weights and biases values is already set -with the values loaded from the disk. +Note that when a model is loaded from the disk +(see [previous paragraph](#initialize-links)), its weights' cache is setup +automatically: there is no need to use the `weights` API in this use case. ### Initialize "Hard Resources" -The last thing to do is to initialize the "hard resources". +When the cache for the weights is well setup, +we have to initialize the "hard resources". These are resources that may be time consuming to initialize depending on the size of the model: @@ -234,20 +247,6 @@ be fully loaded into the kernel of the different layers. - GPU mode: the weights, biases... will be uploaded to the GPU device -So now, what would have happened if the cache for weights and biases had -not been set earlier ? - -=> The values for weights would have been initialized "randomly" -while the values for biases would have been initialized to 0. - -To cap it all, the `weights` API is not necessary in the following situations: - -- The model has been loaded from the disk -- We want to train a model from scratch - -But the `initKernel` API is always necessary for the model to be ready to -train/run. - ## Model Transformation In some scenario, we need to transform the model and preserve the diff --git a/Docs/Examples/AutoEncoder.md b/Docs/Examples/AutoEncoder.md new file mode 100644 index 00000000..eb9b1451 --- /dev/null +++ b/Docs/Examples/AutoEncoder.md @@ -0,0 +1,70 @@ +# 🚀 Auto Encoder Example + +This is the documentation of a +[toy Auto Encoder model](../../Tests/GrAIExamples/AutoEncoderExample.swift), +trained on the GPU. +The dataset used is CIFAR 10. + +We want to train the model to encode and generate images of ships (label 8). + +Here is a subset of the data input images. + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +## Setup + +This example has some `Python` dependencies. In order to run +the example, we first have to setup the environment: + +```bash +conda create --name graiexamples python=3.9 +conda activate graiexamples +cd Tests/GrAIExamples/Base +pip install -e . +``` + +Now, let us run the tests from Xcode or a `bash` command (here with compiler +optimization): + +```bash +swift test -c release --filter GrAIExamples +``` + +It is finally possible to clean the environment 🌍 + +```bash +conda deactivate +conda env remove --name graiexamples +``` + +## Steps + +1. Dump the training dataset. +1. Train a simple auto encoder model. +1. Train a UNet like auto encoder model. +1. Train a StyleGAN like auto encoder model. diff --git a/Docs/Examples/EXAMPLES.md b/Docs/Examples/EXAMPLES.md index 862b5050..21f388b8 100644 --- a/Docs/Examples/EXAMPLES.md +++ b/Docs/Examples/EXAMPLES.md @@ -9,4 +9,6 @@ or in the [GitHub](https://github.com/owkin/GrAIdient/actions) CI The following examples are currently available: -- [VGGExample](VGG.md) +- [VGG](VGG.md) +- [Vision Transformer](VisionTransformer.md) +- [Auto Encoder](AutoEncoder.md) diff --git a/Docs/Examples/VGG.md b/Docs/Examples/VGG.md index 8b9e201d..40f3db74 100644 --- a/Docs/Examples/VGG.md +++ b/Docs/Examples/VGG.md @@ -11,48 +11,48 @@ Here is a subset of images we find for the label 8 (ships) vs label 5 (dogs). - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - - + + + +
diff --git a/Docs/Examples/VisionTransformer.md b/Docs/Examples/VisionTransformer.md new file mode 100644 index 00000000..6dfdf405 --- /dev/null +++ b/Docs/Examples/VisionTransformer.md @@ -0,0 +1,88 @@ +# 🚀 Vision Transformer Example + +This is the documentation of a [toy Vision Transformer model](../../Tests/GrAIExamples/TransformerExample.swift), +trained on the GPU. +The dataset used is CIFAR 10. + +We want to train the model to discriminate between 2 labels +(not the 10 available of the CIFAR 10 dataset): the 8 (ships) or the 5 (dogs). + +Here is a subset of images we find for the label 8 (ships) vs label 5 (dogs). + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +## Setup + +This example has some `Python` dependencies. In order to run +the example, we first have to setup the environment: + +```bash +conda create --name graiexamples python=3.9 +conda activate graiexamples +cd Tests/GrAIExamples/Base +pip install -e . +``` + +Now, let us run the tests from Xcode or a `bash` command (here with compiler +optimization): + +```bash +swift test -c release --filter GrAIExamples +``` + +It is finally possible to clean the environment 🌍 + +```bash +conda deactivate +conda env remove --name graiexamples +``` + +## Steps + +1. Dump the training dataset. +1. Train a simple Vision Transformer model. diff --git a/Package.swift b/Package.swift index c2bc3287..8cc64efb 100644 --- a/Package.swift +++ b/Package.swift @@ -49,7 +49,6 @@ let package = Package( name: "GrAIExamples", dependencies: ["GrAIdient", "PythonKit"], resources: [ - .copy("Base/data"), .copy("Base/python_lib"), .copy("Base/setup.py") ] diff --git a/Sources/GrAITestsUtils/Trainer.swift b/Sources/GrAITestsUtils/Trainer.swift index 9475c6b9..74a85820 100644 --- a/Sources/GrAITestsUtils/Trainer.swift +++ b/Sources/GrAITestsUtils/Trainer.swift @@ -64,6 +64,28 @@ extension TestError: CustomStringConvertible } } +/// +/// Set one weight initialization scheme between the ones available. +/// +/// - Parameter model: The model on which to select the initialization scheme. +/// +func randomSelectWeightsInitializationScheme(model: Model) +{ + let choice = Int.random(in: 0...4) + switch choice { + case 1: + model.weightInitClass = .XavierUniform + case 2: + model.weightInitClass = .XavierNormal + case 3: + model.weightInitClass = .KaimingUniform + case 4: + model.weightInitClass = .KaimingNormal + default: + break + } +} + /// /// Function used to retry flaky numeric tests. /// @@ -162,6 +184,7 @@ open class GradTrainer: Trainer getGradientsApprox: (LossT, Model)->[Double], validate: (Double) throws -> ()) throws { + randomSelectWeightsInitializationScheme(model: model) model.initialize( params: optimizerParams, phase: .Training, @@ -256,6 +279,7 @@ open class FlowTrainer: Trainer if i == 0 { GrAI.Opti.CPU = true + randomSelectWeightsInitializationScheme(model: modelCPU) } if i > 0 @@ -484,6 +508,40 @@ open class FlowReverseTrainer: FlowTrainer } } +/// Compares gradients of weights computed in the CPU execution context againt the GPU one +/// when we accumulate gradients. +open class FlowAccumulateTrainer: FlowTrainer +{ + /// + /// Run the test. + /// + /// The goal is to compare the gradients of weights computed in the CPU execution context with + /// the gradients of weights computed in the GPU execution context. + /// + /// - Parameters: + /// - setData: A function to create/set data to the model. + /// - setLoss: A function to create/set ground truth to the model. + /// - validate: A function that checks whether the relative difference is small enough. + /// + public override func run( + setData: (DataT?, Model)->(DataT, Int), + setLoss: (LossT?, Model)->(LossT), + validate: (Double) throws -> ()) throws + { + modelCPU.accumulateDeltaWeights = true + modelGPU.accumulateDeltaWeights = true + try super.run(setData: setData, setLoss: setLoss, validate: validate) + + modelCPU.accumulateDeltaWeights = false + modelGPU.accumulateDeltaWeights = false + try super.run(setData: setData, setLoss: setLoss, validate: validate) + + modelCPU.accumulateDeltaWeights = true + modelGPU.accumulateDeltaWeights = true + try super.run(setData: setData, setLoss: setLoss, validate: validate) + } +} + /// Pipeline that compares losses computed in the CPU execution context againt the GPU one /// during the inference phase. open class InferenceTrainer: FlowTrainer diff --git a/Sources/GrAIdient/Core/Function/Activation.swift b/Sources/GrAIdient/Core/Function/Activation.swift index 714c5b84..6171a184 100644 --- a/Sources/GrAIdient/Core/Function/Activation.swift +++ b/Sources/GrAIdient/Core/Function/Activation.swift @@ -13,7 +13,8 @@ let ACTIVATION_REGISTRY: [String: Codable.Type] = buildRegistry( ReLU.self, LeakyReLU.self, SoftReLU.self, - Sigmoid.self + Sigmoid.self, + GELU.self ]) /// Activation function to be used in a layer. @@ -37,6 +38,18 @@ open class ActivationFunction: Codable } } + /// + /// Coefficient to apply during the weights initialization. + /// + /// - Returns: The coefficient. + /// + open var coeffInitWeights: Float + { + get { + return 1.0 + } + } + private enum Keys: String, CodingKey { case name @@ -83,18 +96,6 @@ open class ActivationFunction: Codable try container.encode(name, forKey: .name) } - /// - /// Coefficient to apply during the weights initialization. - /// - /// - Parameters: - /// - nPrev: The number of input connections. - /// - nCur: The number of output connections. - /// - open func coeffInitWeights(nPrev: Int, nCur: Int) -> Double - { - return sqrt(2.0 / Double(nPrev + nCur)) - } - /// /// Forward CPU. /// @@ -162,6 +163,28 @@ open class ActivationFunction: Codable }}}} } + /// + /// Forward Gradient Checking CPU. + /// + /// - Parameter layer: Layer to execute the activation function for. + /// + func forwardGC(_ layer: ActivationSeq) + { + let nbBatch = layer.batchSize + let neurons = layer.neurons! + let nbGC = layer.nbGC + + for neuron in neurons.all { + for batch in 0..( + nbElems, deviceID: layer.deviceID) + } + _forwardGPU( + tmp: layer._tmp, + outs: layer.outs, + deviceID: layer.deviceID + ) + } + /// /// Backward GPU. /// @@ -359,6 +439,20 @@ open class ActivationFunction: Codable deviceID: layer.deviceID ) } + + /// + /// Backward GPU. + /// + /// - Parameter layer: Layer to execute the activation function for. + /// + open func backwardGPU(_ layer: ActivationSeq) + { + _backwardGPU( + tmp: layer._tmp, + delta: layer.delta, + deviceID: layer.deviceID + ) + } } /// ReLU activation function. @@ -381,6 +475,18 @@ public class ReLU: ActivationFunction } } + /// + /// Coefficient to apply during the weights initialization. + /// + /// - Returns: The coefficient. + /// + open override var coeffInitWeights: Float + { + get { + return sqrt(2.0) + } + } + /// Create a ReLU activation function. init() { @@ -400,18 +506,6 @@ public class ReLU: ActivationFunction try super.init(from: decoder) } - /// - /// Coefficient to apply during the weights initialization. - /// - /// - Parameters: - /// - nPrev: The number of input connections. - /// - nCur: The number of output connections. - /// - public override func coeffInitWeights(nPrev: Int, nCur: Int) -> Double - { - return sqrt(2.0 / Double(nPrev)) - } - /// /// Forward CPU. /// @@ -465,6 +559,18 @@ public class LeakyReLU: ActivationFunction } } + /// + /// Coefficient to apply during the weights initialization. + /// + /// - Returns: The coefficient. + /// + open override var coeffInitWeights: Float + { + get { + return Float(sqrt(2.0 / (1 + Ɛ * Ɛ))) + } + } + /// Create a LeakyReLU activation function. init() { @@ -484,18 +590,6 @@ public class LeakyReLU: ActivationFunction try super.init(from: decoder) } - /// - /// Coefficient to apply during the weights initialization. - /// - /// - Parameters: - /// - nPrev: The number of input connections. - /// - nCur: The number of output connections. - /// - public override func coeffInitWeights(nPrev: Int, nCur: Int) -> Double - { - return sqrt(2.0 / Double(nPrev)) - } - /// /// Forward CPU. /// @@ -549,6 +643,18 @@ public class SoftReLU: ActivationFunction } } + /// + /// Coefficient to apply during the weights initialization. + /// + /// - Returns: The coefficient. + /// + open override var coeffInitWeights: Float + { + get { + return Float(sqrt(2.0 / (1 + Ɛ * Ɛ))) + } + } + /// Create a SoftReLU activation function. init() { @@ -568,18 +674,6 @@ public class SoftReLU: ActivationFunction try super.init(from: decoder) } - /// - /// Coefficient to apply during the weights initialization. - /// - /// - Parameters: - /// - nPrev: The number of input connections. - /// - nCur: The number of output connections. - /// - public override func coeffInitWeights(nPrev: Int, nCur: Int) -> Double - { - return sqrt(2.0 / Double(nPrev)) - } - /// /// Forward CPU. /// @@ -642,16 +736,86 @@ public class Sigmoid: ActivationFunction try super.init(from: decoder) } + /// + /// Forward CPU. + /// + /// - Parameter x: The input. + /// - Returns: The output. + /// + public override func apply(_ x: Double) -> Double + { + if x >= 0 + { + return 1 / (1 + exp(-x)) + } + else + { + return exp(x) / (1 + exp(x)) + } + } + + /// + /// Backward CPU. + /// + /// - Parameter x: The input. + /// - Returns: The output. + /// + public override func derivate(_ x: Double) -> Double + { + let fx = apply(x) + return fx * (1 - fx) + } +} + +/// GELU activation function. +public class GELU: ActivationFunction +{ + public static let str = "GELU" + + /// Forward GPU kernel. + public override var forwardKernel: String + { + get { + return "forwardGELU" + } + } + /// Backward GPU kernel. + public override var backwardKernel: String + { + get { + return "backwardGELU" + } + } + /// /// Coefficient to apply during the weights initialization. /// - /// - Parameters: - /// - nPrev: The number of input connections. - /// - nCur: The number of output connections. + /// - Returns: The coefficient. + /// + open override var coeffInitWeights: Float + { + get { + return Float(sqrt(2.0)) + } + } + + /// Create a GELU activation function. + init() + { + super.init(GELU.str) + } + + /// + /// Decode from the disk. /// - public override func coeffInitWeights(nPrev: Int, nCur: Int) -> Double + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + required public init(from decoder: Decoder) throws { - return sqrt(1.0 / Double(nPrev)) + try super.init(from: decoder) } /// @@ -662,7 +826,18 @@ public class Sigmoid: ActivationFunction /// public override func apply(_ x: Double) -> Double { - return 1 / (1 + exp(-x)) + let cst = sqrt(2.0 / Double.pi) + let tmp1 = cst * (x + 0.044715 * pow(x, 3)) + let tmp2: Double + if tmp1 >= 0 + { + tmp2 = (1.0 - exp(-2.0 * tmp1)) / (1.0 + exp(-2.0 * tmp1)) + } + else + { + tmp2 = (exp(2.0 * tmp1) - 1.0) / (exp(2.0 * tmp1) + 1.0) + } + return 0.5 * x * (1 + tmp2) } /// @@ -673,8 +848,20 @@ public class Sigmoid: ActivationFunction /// public override func derivate(_ x: Double) -> Double { - let fx = apply(x) - return fx * (1 - fx) + let cst = sqrt(2.0 / Double.pi) + let tmp1 = cst * (x + 0.044715 * pow(x, 3)) + let tmp2: Double + if tmp1 >= 0 + { + tmp2 = (1.0 - exp(-2.0 * tmp1)) / (1.0 + exp(-2.0 * tmp1)) + } + else + { + tmp2 = (exp(2.0 * tmp1) - 1.0) / (exp(2.0 * tmp1) + 1.0) + } + let tmp3 = cst * (1 + 3 * 0.044715 * x * x) * (1 - tmp2 * tmp2) + let derivative = 0.5 * (1 + tmp2 + x * tmp3) + return derivative } } @@ -698,7 +885,8 @@ class ActivationKernelImpl: ActivationKernel ReLU.str: ReLUKernel(), LeakyReLU.str: LeakyReLUKernel(), SoftReLU.str: SoftReLUKernel(), - Sigmoid.str: SigmoidKernel() + Sigmoid.str: SigmoidKernel(), + GELU.str: GELUKernel() ] /// @@ -765,3 +953,13 @@ private class SigmoidKernel: ActivationKernelImpl return Sigmoid() } } + +/// Factory to build a Sigmoid function. +private class GELUKernel: ActivationKernelImpl +{ + /// Build a Sigmoid function. + override func build() -> ActivationFunction + { + return GELU() + } +} diff --git a/Sources/GrAIdient/Core/Function/Normalization.swift b/Sources/GrAIdient/Core/Function/Normalization.swift index b587c6de..8a5e40b8 100644 --- a/Sources/GrAIdient/Core/Function/Normalization.swift +++ b/Sources/GrAIdient/Core/Function/Normalization.swift @@ -33,6 +33,27 @@ class Normalization let outsNew = vDSP.add(β, vDSP.multiply(Ɣ, xHat)) return outsNew } + + /// + /// Forward Gradient Checking LayerNorm CPU. + /// + /// - Parameters: + /// - outs: The data to normalize. + /// - β: The biases to add to the normalization result. + /// - Ɣ: The weights to scale the normalization result. + /// - Returns: The data normalized. + /// + static func forwardGC(outs: [Double], + β: [Double], + Ɣ: [Double]) -> [Double] + { + let μ = vDSP.mean(outs) + let tmp1 = vDSP.add(-μ, outs) + let σ2 = vDSP.meanSquare(tmp1) + let xHat = vDSP.divide(tmp1, sqrt(σ2 + _Ɛ)) + let outsNew = vDSP.add(β, vDSP.multiply(Ɣ, xHat)) + return outsNew + } /// /// Forward Training CPU. @@ -65,6 +86,38 @@ class Normalization μ: μ, σ2: σ2) } + + /// + /// Forward LayerNorm CPU. + /// + /// - Parameters: + /// - outs: The data to normalize. + /// - β: The biases to add to the normalization result. + /// - Ɣ: The weights to scale the normalization result. + /// - Returns: (The data normalized, + /// The data normalized without taking into account the bias and the weight, + /// The average of the data, + /// The deviation of the data). + /// + static func forward(outs: [Double], + β: [Double], + Ɣ: [Double]) -> (outsNew: [Double], + xHat: [Double], + μ: Double, + σ2: Double) + { + + let μ = vDSP.mean(outs) + let tmp1 = vDSP.add(-μ, outs) + let σ2 = vDSP.meanSquare(tmp1) + let xHat = vDSP.divide(tmp1, sqrt(σ2 + _Ɛ)) + let outsNew = vDSP.add(β, vDSP.multiply(Ɣ, xHat)) + + return (outsNew: outsNew, + xHat: xHat, + μ: μ, + σ2: σ2) + } /// /// Forward Inference CPU. @@ -129,6 +182,39 @@ class Normalization dβ: dβ, dƔ: dƔ) } + + /// + /// Backward LayerNorm CPU. + /// + /// - Parameters: + /// - delta: The gradients to back propagate. + /// - xHat: The data normalized without taking into account the bias and the weight. + /// - σ2: The deviation of the data. + /// - Ɣ: The weights that scaled the normalization result. + /// - Returns: (The gradient taking into account the normalization, + /// The gradient of β, + /// The gradient of Ɣ). + /// + static func backward(delta: [Double], + xHat: [Double], + σ2: Double, + Ɣ: [Double]) -> [Double] + { + let nbElems = delta.count + let factor = 1.0 / (Double(nbElems) * sqrt(σ2 + _Ɛ)) + + let Ɣdelta = vDSP.multiply(Ɣ, delta) + let sum1 = vDSP.sum(Ɣdelta) + let sum2 = vDSP.sum(vDSP.multiply(Ɣdelta, xHat)) + + let tmp1 = vDSP.add( + multiplication: (Ɣdelta, Double(nbElems)), + multiplication: (xHat, -sum2)) + let deltaNew = vDSP.add( + multiplication: (tmp1, factor), -factor * sum1) + + return deltaNew + } /// /// Backward Inference CPU. diff --git a/Sources/GrAIdient/Core/Layer/Layer.swift b/Sources/GrAIdient/Core/Layer/Layer.swift index 8e4640a1..34dd42f6 100644 --- a/Sources/GrAIdient/Core/Layer/Layer.swift +++ b/Sources/GrAIdient/Core/Layer/Layer.swift @@ -10,6 +10,8 @@ import Foundation /// Error occuring during the layer forward or backward propagation. public enum LayerError: Error { + /// Error during the initialization of a layer. + case Init(message: String) /// Data has not the correct dimensions. case DataSize /// Batch size is not coherent. @@ -22,6 +24,8 @@ extension LayerError: CustomStringConvertible { switch self { + case .Init(let message): + return message case .DataSize: return "The parameters do not have the expected number of elements." case .BatchSize: @@ -91,7 +95,7 @@ open class Layer: Codable /// Whether the gradient has been updated or not. public var dirty = true - /// Whether to compute gradients of not. + /// Whether to compute gradients or not. public var computeDelta = true /// Slight modification to use during gradient checking. @@ -222,6 +226,11 @@ open class Layer: Codable /// open func initLinks(_ layers: [Layer]) { + if idPrev < 0 + { + layerPrev = nil + return + } for testLayer in layers { if testLayer.id == idPrev diff --git a/Sources/GrAIdient/Core/Layer/LayerNormalization.swift b/Sources/GrAIdient/Core/Layer/LayerNormalization.swift index 53b57e1f..3154be8c 100644 --- a/Sources/GrAIdient/Core/Layer/LayerNormalization.swift +++ b/Sources/GrAIdient/Core/Layer/LayerNormalization.swift @@ -12,18 +12,16 @@ public protocol Cloneable func clone() -> Self } -/// A layer that applies batch normalization. -public class BatchNormalizationBase: Codable, Cloneable +/// A layer that applies normalization, containing weights. +public class LayerWeightsNormalization: Codable, Cloneable { - /// Number of independent units of batch normalization. + /// Number of independent units of normalization. let _nbNeurons: Int - /// Number of elements in one batch size. + /// Number of elements in one normalization unit. var _nbElems = 0 /// Cache for weights before calling `initKernel` API. var _weightsList = [Float]() - /// Cache for stats before calling `initKernel` API. - var _statsList = [Float]() /// Weights in the CPU execution context. var weights: [Float] @@ -35,27 +33,16 @@ public class BatchNormalizationBase: Codable, Cloneable _weightsList = newValue } } - /// Stats in the CPU execution context. - var stats: [Float] - { - get { - return _statsList - } - set { - _statsList = newValue - } - } private enum Keys: String, CodingKey { case nbNeurons case nbElems case weights - case stats } /// - /// Create a layer with independent units of batch normalization. + /// Create a layer with independent units of normalization. /// /// - Parameter nbNeurons: Number of independent units. /// @@ -65,15 +52,45 @@ public class BatchNormalizationBase: Codable, Cloneable } /// - /// Create a layer with independent units of batch normalization. + /// Create a layer with independent units of normalization. /// - /// - Parameter layer: The layer with the structure we want to apply the batch normalization to . + /// - Parameter layer: The layer with the structure we want to apply the normalization to . /// convenience init(_ layer: BN2D) { self.init(nbNeurons: layer.nbChannels) } + /// + /// Create a layer with independent units of normalization. + /// + /// - Parameter layer: The layer with the structure we want to apply the normalization to . + /// + convenience init(_ layer: InstanceNorm2D) + { + self.init(nbNeurons: layer.nbChannels) + } + + /// + /// Create a layer with independent units of normalization. + /// + /// - Parameter layer: The layer with the structure we want to apply the normalization to . + /// + convenience init(_ layer: AdaIN) + { + self.init(nbNeurons: layer.nbChannels) + } + + /// + /// Create a layer with independent units of normalization. + /// + /// - Parameter layer: The layer with the structure we want to apply the normalization to . + /// + convenience init(_ layer: LayerNormSeq) + { + self.init(nbNeurons: layer.nbNeurons) + } + /// /// Decode from the disk. /// @@ -90,30 +107,26 @@ public class BatchNormalizationBase: Codable, Cloneable _nbElems = try container.decode(Int.self, forKey: .nbElems) _weightsList = try container.decode([Float].self, forKey: .weights) - _statsList = try container.decode([Float].self, forKey: .stats) } /// - /// Copy a batch normalization layer. + /// Copy a normalization layer. /// - /// - Parameter bn: The layer to copy. + /// - Parameter norm: The layer to copy. /// - init(bn: BatchNormalizationBase) + init(norm: LayerWeightsNormalization) { - _nbNeurons = bn._nbNeurons - _nbElems = bn._nbElems + _nbNeurons = norm._nbNeurons + _nbElems = norm._nbElems - let weights = bn.weights + let weights = norm.weights self.weights = weights - - let stats = bn.stats - self.stats = stats } /// Copy this. public func clone() -> Self { - return BatchNormalizationBase(bn: self) as! Self + return LayerWeightsNormalization(norm: self) as! Self } /// @@ -135,14 +148,110 @@ public class BatchNormalizationBase: Codable, Cloneable try container.encode(_nbElems, forKey: .nbElems) let weights = self.weights - let stats = self.stats try container.encode(weights, forKey: .weights) + } +} + +/// A layer that applies batch normalization, containing weights and stats. +public class LayerWeightsStatsNormalization: LayerWeightsNormalization +{ + /// Cache for stats before calling `initKernel` API. + var _statsList = [Float]() + + /// Stats in the CPU execution context. + var stats: [Float] + { + get { + return _statsList + } + set { + _statsList = newValue + } + } + + private enum Keys: String, CodingKey + { + case stats + } + + /// + /// Create a layer with independent units of normalization. + /// + /// - Parameter nbNeurons: Number of independent units. + /// + override init(nbNeurons: Int) + { + super.init(nbNeurons: nbNeurons) + } + + /// + /// Create a layer with independent units of normalization. + /// + /// - Parameter layer: The layer with the structure we want to apply the normalization to . + /// + convenience init(_ layer: BN2D) + { + self.init(nbNeurons: layer.nbChannels) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let container = try decoder.container(keyedBy: Keys.self) + _statsList = try container.decode([Float].self, forKey: .stats) + try super.init(from: decoder) + } + + /// + /// Copy a normalization layer. + /// + /// - Parameter norm: The layer to copy. + /// + init(norm: LayerWeightsStatsNormalization) + { + super.init(norm: norm) + + let stats = norm.stats + self.stats = stats + } + + /// Copy this. + public override func clone() -> Self + { + return LayerWeightsStatsNormalization(norm: self) as! Self + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + + let stats = self.stats try container.encode(stats, forKey: .stats) + + try super.encode(to: encoder) } } /// A layer that applies batch normalization in the CPU execution context. -public class BatchNormalization: BatchNormalizationBase +public class BatchNormalization: LayerWeightsStatsNormalization { /// Slight modification to avoid "divide by 0" errors. let _Ɛ: Double = 1e-5 @@ -247,7 +356,7 @@ public class BatchNormalization: BatchNormalizationBase /// Copy this. public override func clone() -> Self { - return BatchNormalization(bn: self) as! Self + return BatchNormalization(norm: self) as! Self } /// @@ -392,7 +501,7 @@ public class BatchNormalization: BatchNormalizationBase /// Apply the forward pass of the Gradient Checking in GPU execution context. func forwardFlowGC(_ layer: BN2D) { - layer._bnGPU?.applyWeights(bn: self) + layer._normGPU?.applyWeights(norm: self) forwardGC(layer) } @@ -405,7 +514,7 @@ public class BatchNormalization: BatchNormalizationBase { _Eμ.withUnsafeMutableBufferPointer { EμPointer in _Eσ2.withUnsafeMutableBufferPointer { Eσ2Pointer in - _xHat.withUnsafeMutableBufferPointer { xhatPointer in + _xHat.withUnsafeMutableBufferPointer { xHatPointer in _σ2.withUnsafeMutableBufferPointer { σ2Pointer in Concurrency.slice(nbChannels) { @@ -418,7 +527,7 @@ public class BatchNormalization: BatchNormalizationBase ) layer.setOuts(depth: depth, outs: outs) - xhatPointer[depth] = xHat + xHatPointer[depth] = xHat σ2Pointer[depth] = σ2 if _nbElems == 0 @@ -518,7 +627,7 @@ public class BatchNormalization: BatchNormalizationBase } /// A layer that applies batch normalization in the GPU execution context. -class BatchNormalizationGPU: BatchNormalizationBase +class BatchNormalizationGPU: LayerWeightsStatsNormalization { /// /// Buffer of weights to scale the normalization result. @@ -629,7 +738,7 @@ class BatchNormalizationGPU: BatchNormalizationBase /// Copy this. public override func clone() -> Self { - return BatchNormalizationGPU(bn: self) as! Self + return BatchNormalizationGPU(norm: self) as! Self } /// @@ -731,15 +840,15 @@ class BatchNormalizationGPU: BatchNormalizationBase /// /// This function is necessary for the Gradient Checking in the GPU execution context. /// - /// - Parameter bn: The layer in the CPU execution context. + /// - Parameter norm: The layer in the CPU execution context. /// - func applyWeights(bn: BatchNormalization) + func applyWeights(norm: BatchNormalization) { let weights = self.weights for depth in 0..<_nbNeurons { - bn._Ɣ.w[depth] = Double(weights[depth]) - bn._β.w[depth] = Double(weights[_nbNeurons + depth]) + norm._Ɣ.w[depth] = Double(weights[depth]) + norm._β.w[depth] = Double(weights[_nbNeurons + depth]) } } @@ -775,7 +884,7 @@ class BatchNormalizationGPU: BatchNormalizationBase } let command = MetalKernel.get.createCommand( - "computeConvμ", deviceID: _deviceID + "computeBNConvμ", deviceID: _deviceID ) command.setBuffer(layer.outs.metal, atIndex: 0) command.setBytes(pNbChannels, atIndex: 1) @@ -808,7 +917,7 @@ class BatchNormalizationGPU: BatchNormalizationBase } let command = MetalKernel.get.createCommand( - "computeConvσ2", deviceID: _deviceID + "computeBNConvσ2", deviceID: _deviceID ) command.setBuffer(layer.outs.metal, atIndex: 0) command.setBuffer(_μ.metal, atIndex: 1) @@ -1022,3 +1131,1585 @@ class BatchNormalizationGPU: BatchNormalizationBase return [_Ɣ, _β] } } + +/// A layer that applies instance normalization in the CPU execution context. +public class InstanceNormalization: LayerWeightsNormalization +{ + /// Slight modification to avoid "divide by 0" errors. + let _Ɛ: Double = 1e-5 + + /// + /// Array of weights to scale the normalization result. + /// Shape ~ (nbNeurons,). + /// + var _Ɣ: WeightArrays! = nil + /// + /// Array of biases to add to the normalization result. + /// Shape ~ (nbNeurons,). + /// + var _β: WeightArrays! = nil + + /// + /// List of deviations of data for the different independent batch normalization units. + /// Shape ~ ((batch x nbNeurons),). + /// + var _σ2 = [Double]() + + /// + /// The list of data normalized without taking into account the biases and the weights. + /// Shape ~ ((batch x nbNeurons), (height x width)). + /// + var _xHat = [[Double]]() + + /// Weights in the CPU execution context. + override var weights: [Float] + { + get { + if _Ɣ == nil + { + return super.weights + } + + var weightsTmp = [Float]() + for Ɣ in _Ɣ.w + { + weightsTmp.append(Float(Ɣ)) + } + for β in _β.w + { + weightsTmp.append(Float(β)) + } + return weightsTmp + } + set { + if newValue.count > 0 && newValue.count != 2 * _nbNeurons + { + fatalError( + "Weights do not have the expected number of elements." + ) + } + super.weights = newValue + } + } + + /// Copy this. + public override func clone() -> Self + { + return InstanceNormalization(norm: self) as! Self + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We do not clean Ɣ and β but must reset their momentum state. + /// Note that we do not have to reset their delta because here they are independent on + /// batch size. + /// + func resetKernel() + { + _σ2 = [] + _xHat = [] + + _Ɣ?.reset() + _β?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum state is also reset. + /// Note that we also initialize the delta which are independent on the batch size. + /// + func initWeights() + { + _β = WeightArrays(_nbNeurons) + _Ɣ = WeightArrays(_nbNeurons) + if _weightsList.count == 0 + { + for depth in 0..<_nbNeurons + { + _Ɣ.w[depth] = 1.0 + _β.w[depth] = 0.0 + } + } + else + { + for depth in 0..<_nbNeurons + { + _Ɣ.w[depth] = Double(_weightsList[depth]) + _β.w[depth] = Double(_weightsList[_nbNeurons + depth]) + } + _weightsList = [] + } + } + + /// Apply the forward pass of the Gradient Checking in CPU execution context. + func forwardGC(_ layer: InstanceNorm2D) + { + let nbGC = layer.nbGC + let nbChannels = layer.nbChannels + let Ɛ = layer.Ɛ + + Concurrency.slice(nbChannels) + { + (depth: Int) in + + for batch in 0..= nbGC-4*nbChannels && elem < nbGC-2*nbChannels && + depth == (elem-nbGC+4*nbChannels)/2 + { + if elem % 2 == 0 + { + outs = Normalization.forwardGC( + outs: layer.getOutsGC( + depth: depth, batch: batch, elem: elem + ), + β: _β.w[depth], + Ɣ: _Ɣ.w[depth]+Ɛ + ) + } + else + { + outs = Normalization.forwardGC( + outs: layer.getOutsGC( + depth: depth, batch: batch, elem: elem + ), + β: _β.w[depth], + Ɣ: _Ɣ.w[depth]-Ɛ + ) + } + } + else if elem >= nbGC-2*nbChannels && + depth == (elem-nbGC+2*nbChannels)/2 + { + if elem % 2 == 0 + { + outs = Normalization.forwardGC( + outs: layer.getOutsGC( + depth: depth, batch: batch, elem: elem + ), + β: _β.w[depth]+Ɛ, + Ɣ: _Ɣ.w[depth] + ) + } + else + { + outs = Normalization.forwardGC( + outs: layer.getOutsGC( + depth: depth, batch: batch, elem: elem + ), + β: _β.w[depth]-Ɛ, + Ɣ: _Ɣ.w[depth] + ) + } + } + else + { + outs = Normalization.forwardGC( + outs: layer.getOutsGC( + depth: depth, batch: batch, elem: elem + ), + β: _β.w[depth], + Ɣ: _Ɣ.w[depth] + ) + } + layer.setOutsGC( + depth: depth, batch: batch, elem: elem, outs: outs + ) + }} + } + } + + /// Apply the forward pass of the Gradient Checking in GPU execution context. + func forwardFlowGC(_ layer: InstanceNorm2D) + { + layer._normGPU?.applyWeights(norm: self) + forwardGC(layer) + } + + /// Apply the forward pass in the CPU execution context. + func forward(_ layer: InstanceNorm2D) + { + if _σ2.count == 0 + { + _σ2 = [Double]( + repeating: 0.0, count: _nbNeurons * layer.batchSize + ) + _xHat = [[Double]]( + repeating: [], count: _nbNeurons * layer.batchSize + ) + } + + let nbChannels = layer.nbChannels + + _xHat.withUnsafeMutableBufferPointer { xHatPointer in + _σ2.withUnsafeMutableBufferPointer { σ2Pointer in + Concurrency.slice(nbChannels) + { + (depth: Int) in + + for batch in 0.. [IWeightArrays] + { + return [_Ɣ, _β] + } +} + +/// A layer that applies instance normalization in the GPU execution context. +class InstanceNormalizationGPU: LayerWeightsNormalization +{ + /// + /// Buffer of weights to scale the normalization result. + /// Shape ~ (nbNeurons,). + /// + var _Ɣ: IWeightBuffers! = nil + /// + /// Buffer of biases to add to the normalization result. + /// Shape ~ (nbNeurons,). + /// + var _β: IWeightBuffers! = nil + + /// + /// Buffer of averages of data for the different independent batch normalization units. + /// Shape ~ (batch, nbNeurons). + /// + var _μ: MetalBuffer! = nil + /// + /// Buffer of deviations of data for the different independent batch normalization units. + /// Shape ~ (batch, nbNeurons). + /// + var _σ2: MetalBuffer! = nil + + /// + /// Buffer of data normalized without taking into account the biases and the weights. + /// Shape ~ (batch, nbNeurons, height, width). + /// + var _xHat: MetalBuffer! = nil + + /// + /// Buffer used to compute backward pass. + /// Shape ~ (nbNeurons,). + /// + var _sum1: MetalBuffer! = nil + /// + /// Buffer used to compute backward pass. + /// Shape ~ (nbNeurons,). + /// + var _sum2: MetalBuffer! = nil + + /// GPU device on which model is executed. + var _deviceID = 0 + + /// Weights in the GPU execution context. + override var weights: [Float] + { + get { + if _Ɣ == nil + { + return super.weights + } + + MetalKernel.get.download([_β.w_p!, _Ɣ.w_p!]) + + var weightsTmp = [Float]() + weightsTmp += _Ɣ.w_p!.shared.array + weightsTmp += _β.w_p!.shared.array + return weightsTmp + } + set { + if newValue.count > 0 && newValue.count != 2 * _nbNeurons + { + fatalError( + "Weights do not have the expected number of elements." + ) + } + super.weights = newValue + } + } + + /// Copy this. + public override func clone() -> Self + { + return InstanceNormalizationGPU(norm: self) as! Self + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We do not clean Ɣ and β but must reset their momentum state. + /// + func resetKernel() + { + _σ2 = nil + _xHat = nil + _sum1 = nil + _sum2 = nil + + _Ɣ?.reset() + _β?.reset() + } + + /// + /// Initialize hard resources in the GPU execution context. + /// + /// We initialize the stats. + /// + /// - Parameter deviceID: The id of GPU where to run the model. + /// + func initKernel(deviceID: Int) + { + _deviceID = deviceID + } + + /// + /// Initialize weights in the GPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + func initWeights() + { + _β = WeightBuffers(nbElems: _nbNeurons, deviceID: _deviceID) + _Ɣ = WeightBuffers(nbElems: _nbNeurons, deviceID: _deviceID) + + let βPtr = _β.w_p!.shared.buffer + let ƔPtr = _Ɣ.w_p!.shared.buffer + + if _weightsList.count == 0 + { + for depth in 0..<_nbNeurons + { + ƔPtr[depth] = 1.0 + βPtr[depth] = 0.0 + } + } + else + { + for depth in 0..<_nbNeurons + { + ƔPtr[depth] = _weightsList[depth] + βPtr[depth] = _weightsList[_nbNeurons + depth] + } + _weightsList = [] + } + + MetalKernel.get.upload([_β.w_p!, _Ɣ.w_p!]) + } + + /// + /// Get the weights and biases back to the CPU execution context. + /// + /// This function is necessary for the Gradient Checking in the GPU execution context. + /// + /// - Parameter norm: The layer in the CPU execution context. + /// + func applyWeights(norm: InstanceNormalization) + { + let weights = self.weights + for depth in 0..<_nbNeurons + { + norm._Ɣ.w[depth] = Double(weights[depth]) + norm._β.w[depth] = Double(weights[_nbNeurons + depth]) + } + } + + /// Apply the forward pass in the GPU execution context. + func forward(_ layer: InstanceNorm2D) + { + _computeμ(layer) + _computeσ2(layer) + + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + + if _xHat == nil + { + _xHat = MetalPrivateBuffer( + batchSize * _nbNeurons * width * height, + deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "forwardInstanceNormConv", deviceID: _deviceID + ) + command.setBuffer(_β.w.metal, atIndex: 0) + command.setBuffer(_Ɣ.w.metal, atIndex: 1) + command.setBuffer(_μ.metal, atIndex: 2) + command.setBuffer(_σ2.metal, atIndex: 3) + command.setBytes(pNbChannels, atIndex: 4) + command.setBytes(pNbBatch, atIndex: 5) + command.setBytes(pDimensions, atIndex: 6) + command.setBuffer(layer.outs.metal, atIndex: 7) + command.setBuffer(_xHat.metal, atIndex: 8) + + command.dispatchThreads( + width: _nbNeurons * width, + height: batchSize * height + ) + command.enqueue() + } + + /// Apply the forward pass in the GPU execution context. + func forward(_ layer: AdaIN) + { + _computeμ(layer) + _computeσ2(layer) + + let layerFirst = layer._layersPrev.first as! Layer2D + let layerLast = layer._layersPrev.last as! Layer1D + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + + if _xHat == nil + { + _xHat = MetalPrivateBuffer( + batchSize * _nbNeurons * width * height, + deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "forwardAdaIN", deviceID: _deviceID + ) + command.setBuffer(layerFirst.outs.metal, atIndex: 0) + command.setBuffer(layerLast.outs.metal, atIndex: 1) + command.setBuffer(_μ.metal, atIndex: 2) + command.setBuffer(_σ2.metal, atIndex: 3) + command.setBytes(pNbChannels, atIndex: 4) + command.setBytes(pNbBatch, atIndex: 5) + command.setBytes(pDimensions, atIndex: 6) + command.setBuffer(layer.outs.metal, atIndex: 7) + command.setBuffer(_xHat.metal, atIndex: 8) + + command.dispatchThreads( + width: _nbNeurons * width, + height: batchSize * height + ) + command.enqueue() + } + + /// Compute the averages of the different independent normalization units. + private func _computeμ(_ layer: InstanceNorm2D) + { + let nbChannels = layer.nbChannels + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + + if _μ == nil + { + _μ = MetalPrivateBuffer( + batchSize * _nbNeurons, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "computeInstanceNormConvμ", deviceID: _deviceID + ) + command.setBuffer(layer.outs.metal, atIndex: 0) + command.setBytes(pNbChannels, atIndex: 1) + command.setBytes(pNbBatch, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBuffer(_μ.metal, atIndex: 4) + + command.dispatchThreads(width: _nbNeurons, height: batchSize) + command.enqueue() + } + + /// Compute the averages of the different independent normalization units. + private func _computeμ(_ layer: AdaIN) + { + let layerFirst = layer._layersPrev.first as! Layer2D + let nbChannels = layer.nbChannels + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + + if _μ == nil + { + _μ = MetalPrivateBuffer( + batchSize * _nbNeurons, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "computeInstanceNormConvμ", deviceID: _deviceID + ) + command.setBuffer(layerFirst.outs.metal, atIndex: 0) + command.setBytes(pNbChannels, atIndex: 1) + command.setBytes(pNbBatch, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBuffer(_μ.metal, atIndex: 4) + + command.dispatchThreads(width: _nbNeurons, height: batchSize) + command.enqueue() + } + + /// Compute the deviations of the different independent normalization units. + private func _computeσ2(_ layer: InstanceNorm2D) + { + let nbChannels = layer.nbChannels + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + + if _σ2 == nil + { + _σ2 = MetalPrivateBuffer( + batchSize * _nbNeurons, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "computeInstanceNormConvσ2", deviceID: _deviceID + ) + command.setBuffer(layer.outs.metal, atIndex: 0) + command.setBuffer(_μ.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBytes(pDimensions, atIndex: 4) + command.setBuffer(_σ2.metal, atIndex: 5) + + command.dispatchThreads(width: _nbNeurons, height: batchSize) + command.enqueue() + } + + /// Compute the deviations of the different independent normalization units. + private func _computeσ2(_ layer: AdaIN) + { + let layerFirst = layer._layersPrev.first as! Layer2D + let nbChannels = layer.nbChannels + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + + if _σ2 == nil + { + _σ2 = MetalPrivateBuffer( + batchSize * _nbNeurons, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "computeInstanceNormConvσ2", deviceID: _deviceID + ) + command.setBuffer(layerFirst.outs.metal, atIndex: 0) + command.setBuffer(_μ.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBytes(pDimensions, atIndex: 4) + command.setBuffer(_σ2.metal, atIndex: 5) + + command.dispatchThreads(width: _nbNeurons, height: batchSize) + command.enqueue() + } + + /// Apply the backward pass in the GPU execution context. + func backward(_ layer: InstanceNorm2D) + { + _backwardWeights(layer) + + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + + let command = MetalKernel.get.createCommand( + "backwardInstanceNormConv", deviceID: _deviceID + ) + command.setBuffer(_σ2.metal, atIndex: 0) + command.setBuffer(_xHat.metal, atIndex: 1) + command.setBuffer(_Ɣ.w.metal, atIndex: 2) + command.setBuffer(_sum1.metal, atIndex: 3) + command.setBuffer(_sum2.metal, atIndex: 4) + command.setBytes(pNbChannels, atIndex: 5) + command.setBytes(pNbBatch, atIndex: 6) + command.setBytes(pDimensions, atIndex: 7) + command.setBuffer(layer.delta.metal, atIndex: 8) + + command.dispatchThreads( + width: _nbNeurons * width, + height: batchSize * height + ) + command.enqueue() + } + + /// Apply the backward pass in the GPU execution context. + func backward(_ layer: AdaIN) + { + _backward(layer) + + let layerFirst = layer._layersPrev.first as! Layer2D + let layerLast = layer._layersPrev.last as! Layer1D + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pDirty: [UInt32] = layerFirst.dirty ? [1] : [0] + + let command = MetalKernel.get.createCommand( + "backward1AdaIN", deviceID: _deviceID + ) + command.setBuffer(layer.delta.metal, atIndex: 0) + command.setBuffer(_σ2.metal, atIndex: 1) + command.setBuffer(_xHat.metal, atIndex: 2) + command.setBuffer(layerLast.outs.metal, atIndex: 3) + command.setBuffer(_sum1.metal, atIndex: 4) + command.setBuffer(_sum2.metal, atIndex: 5) + command.setBytes(pNbChannels, atIndex: 6) + command.setBytes(pNbBatch, atIndex: 7) + command.setBytes(pDimensions, atIndex: 8) + command.setBytes(pDirty, atIndex: 9) + command.setBuffer(layerFirst.delta.metal, atIndex: 10) + + command.dispatchThreads( + width: _nbNeurons * width, + height: batchSize * height + ) + command.enqueue() + } + + /// Compute the gradients of weights in the GPU execution context. + private func _backwardWeights(_ layer: InstanceNorm2D) + { + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pAccumulate: [UInt32] = layer.accumulateDeltaWeights ? [1] : [0] + + if _sum1 == nil + { + _sum1 = MetalPrivateBuffer( + batchSize * _nbNeurons, deviceID: _deviceID + ) + _sum2 = MetalPrivateBuffer( + batchSize * _nbNeurons, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "backwardWeightsInstanceNormConv", deviceID: _deviceID + ) + command.setBuffer(layer.delta.metal, atIndex: 0) + command.setBuffer(_xHat.metal, atIndex: 1) + command.setBuffer(_Ɣ.w.metal, atIndex: 2) + command.setBytes(pNbChannels, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBytes(pDimensions, atIndex: 5) + command.setBytes(pAccumulate, atIndex: 6) + command.setBuffer(_sum1.metal, atIndex: 7) + command.setBuffer(_sum2.metal, atIndex: 8) + command.setBuffer(_Ɣ.g.metal, atIndex: 9) + command.setBuffer(_β.g.metal, atIndex: 10) + + command.dispatchThreads(_nbNeurons) + command.enqueue() + } + + /// Compute the gradients of weights in the GPU execution context. + private func _backward(_ layer: AdaIN) + { + let layerLast = layer._layersPrev.last as! Layer1D + let batchSize = layer.batchSize + let width = layer.width + let height = layer.height + + let pNbChannels: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pDirty: [UInt32] = layerLast.dirty ? [1] : [0] + + if _sum1 == nil + { + _sum1 = MetalPrivateBuffer( + batchSize * _nbNeurons, deviceID: _deviceID + ) + _sum2 = MetalPrivateBuffer( + batchSize * _nbNeurons, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "backward2AdaIN", deviceID: _deviceID + ) + command.setBuffer(layer.delta.metal, atIndex: 0) + command.setBuffer(_xHat.metal, atIndex: 1) + command.setBuffer(layerLast.outs.metal, atIndex: 2) + command.setBytes(pNbChannels, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBytes(pDimensions, atIndex: 5) + command.setBytes(pDirty, atIndex: 6) + command.setBuffer(_sum1.metal, atIndex: 7) + command.setBuffer(_sum2.metal, atIndex: 8) + command.setBuffer(layerLast.delta.metal, atIndex: 9) + + command.dispatchThreads(width: _nbNeurons, height: batchSize) + command.enqueue() + } + + /// Get the weights in the GPU execution context. + func collectWeights() -> [IWeightBuffers] + { + return [_Ɣ, _β] + } +} + +/// A layer that applies layer normalization in the CPU execution context. +public class LayerNormalization: LayerWeightsNormalization +{ + /// Slight modification to avoid "divide by 0" errors. + let _Ɛ: Double = 1e-5 + + /// + /// Array of weights to scale the normalization result. + /// Shape ~ (nbNeurons,). + /// + var _Ɣ: WeightArrays! = nil + /// + /// Array of biases to add to the normalization result. + /// Shape ~ (nbNeurons,). + /// + var _β: WeightArrays! = nil + + /// + /// List of deviations of data for the different independent batch normalization units. + /// Shape ~ ((batch x sequence),). + /// + var _σ2 = [Double]() + + /// + /// The list of data normalized without taking into account the biases and the weights. + /// Shape ~ ((batch x sequence), (nbNeurons)). + /// + var _xHat = [[Double]]() + + /// Weights in the CPU execution context. + override var weights: [Float] + { + get { + if _Ɣ == nil + { + return super.weights + } + + var weightsTmp = [Float]() + for Ɣ in _Ɣ.w + { + weightsTmp.append(Float(Ɣ)) + } + for β in _β.w + { + weightsTmp.append(Float(β)) + } + return weightsTmp + } + set { + if newValue.count > 0 && newValue.count != 2 * _nbNeurons + { + fatalError( + "Weights do not have the expected number of elements." + ) + } + super.weights = newValue + } + } + + /// Copy this. + public override func clone() -> Self + { + return LayerNormalization(norm: self) as! Self + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We do not clean Ɣ and β but must reset their momentum state. + /// Note that we do not have to reset their delta because here they are independent on + /// batch size. + /// + func resetKernel() + { + _σ2 = [] + _xHat = [] + + _Ɣ?.reset() + _β?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum state is also reset. + /// Note that we also initialize the delta which are independent on the batch size. + /// + func initWeights() + { + _β = WeightArrays(_nbNeurons) + _Ɣ = WeightArrays(_nbNeurons) + if _weightsList.count == 0 + { + for depth in 0..<_nbNeurons + { + _Ɣ.w[depth] = 1.0 + _β.w[depth] = 0.0 + } + } + else + { + for depth in 0..<_nbNeurons + { + _Ɣ.w[depth] = Double(_weightsList[depth]) + _β.w[depth] = Double(_weightsList[_nbNeurons + depth]) + } + _weightsList = [] + } + } + + /// Apply the forward pass of the Gradient Checking in CPU execution context. + func forwardGC(_ layer: LayerNormSeq) + { + let nbGC = layer.nbGC + let nbNeurons = layer.nbNeurons + let Ɛ = layer.Ɛ + + Concurrency.slice(layer.sequence) + { + (seq: Int) in + + for batch in 0..= nbGC-4*nbNeurons && elem < nbGC-2*nbNeurons + { + let DEPTH = (elem - nbGC + 4 * nbNeurons) / 2 + + for depth in 0..= nbGC-2*nbNeurons + { + let DEPTH = (elem - nbGC + 2 * nbNeurons) / 2 + + for depth in 0.. [IWeightArrays] + { + return [_Ɣ, _β] + } +} + +/// A layer that applies layer normalization in the GPU execution context. +class LayerNormalizationGPU: LayerWeightsNormalization +{ + /// + /// Buffer of weights to scale the normalization result. + /// Shape ~ (nbNeurons,). + /// + var _Ɣ: IWeightBuffers! = nil + /// + /// Buffer of biases to add to the normalization result. + /// Shape ~ (nbNeurons,). + /// + var _β: IWeightBuffers! = nil + + /// + /// Buffer of averages of data for the different independent batch normalization units. + /// Shape ~ (batch, sequence). + /// + var _μ: MetalBuffer! = nil + /// + /// Buffer of deviations of data for the different independent batch normalization units. + /// Shape ~ (batch, sequence). + /// + var _σ2: MetalBuffer! = nil + + /// + /// Buffer of data normalized without taking into account the biases and the weights. + /// Shape ~ (batch, sequence, nbNeurons). + /// + var _xHat: MetalBuffer! = nil + + /// + /// Buffer used to compute backward pass. + /// Shape ~ (batch, sequence). + /// + var _sum1: MetalBuffer! = nil + /// + /// Buffer used to compute backward pass. + /// Shape ~ (batch, sequence). + /// + var _sum2: MetalBuffer! = nil + + /// GPU device on which model is executed. + var _deviceID = 0 + + /// Weights in the GPU execution context. + override var weights: [Float] + { + get { + if _Ɣ == nil + { + return super.weights + } + + MetalKernel.get.download([_β.w_p!, _Ɣ.w_p!]) + + var weightsTmp = [Float]() + weightsTmp += _Ɣ.w_p!.shared.array + weightsTmp += _β.w_p!.shared.array + return weightsTmp + } + set { + if newValue.count > 0 && newValue.count != 2 * _nbNeurons + { + fatalError( + "Weights do not have the expected number of elements." + ) + } + super.weights = newValue + } + } + + /// Copy this. + public override func clone() -> Self + { + return LayerNormalizationGPU(norm: self) as! Self + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We do not clean Ɣ and β but must reset their momentum state. + /// + func resetKernel() + { + _σ2 = nil + _xHat = nil + _sum1 = nil + _sum2 = nil + + _Ɣ?.reset() + _β?.reset() + } + + /// + /// Initialize hard resources in the GPU execution context. + /// + /// We initialize the stats. + /// + /// - Parameter deviceID: The id of GPU where to run the model. + /// + func initKernel(deviceID: Int) + { + _deviceID = deviceID + } + + /// + /// Initialize weights in the GPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + func initWeights() + { + _β = WeightBuffers(nbElems: _nbNeurons, deviceID: _deviceID) + _Ɣ = WeightBuffers(nbElems: _nbNeurons, deviceID: _deviceID) + + let βPtr = _β.w_p!.shared.buffer + let ƔPtr = _Ɣ.w_p!.shared.buffer + + if _weightsList.count == 0 + { + for depth in 0..<_nbNeurons + { + ƔPtr[depth] = 1.0 + βPtr[depth] = 0.0 + } + } + else + { + for depth in 0..<_nbNeurons + { + ƔPtr[depth] = _weightsList[depth] + βPtr[depth] = _weightsList[_nbNeurons + depth] + } + _weightsList = [] + } + + MetalKernel.get.upload([_β.w_p!, _Ɣ.w_p!]) + } + + /// + /// Get the weights and biases back to the CPU execution context. + /// + /// This function is necessary for the Gradient Checking in the GPU execution context. + /// + /// - Parameter norm: The layer in the CPU execution context. + /// + func applyWeights(norm: LayerNormalization) + { + let weights = self.weights + for depth in 0..<_nbNeurons + { + norm._Ɣ.w[depth] = Double(weights[depth]) + norm._β.w[depth] = Double(weights[_nbNeurons + depth]) + } + } + + /// Apply the forward pass in the GPU execution context. + func forward(_ layer: LayerNormSeq) + { + _computeμ(layer) + _computeσ2(layer) + + let batchSize = layer.batchSize + let sequence = layer.sequence + + let pNbNeurons: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pSequence: [UInt32] = [UInt32(sequence)] + + if _xHat == nil + { + _xHat = MetalPrivateBuffer( + batchSize * sequence * _nbNeurons, + deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "forwardLayerNormSeq", deviceID: _deviceID + ) + command.setBuffer(_β.w.metal, atIndex: 0) + command.setBuffer(_Ɣ.w.metal, atIndex: 1) + command.setBuffer(_μ.metal, atIndex: 2) + command.setBuffer(_σ2.metal, atIndex: 3) + command.setBytes(pNbNeurons, atIndex: 4) + command.setBytes(pNbBatch, atIndex: 5) + command.setBytes(pSequence, atIndex: 6) + command.setBuffer(layer.outs.metal, atIndex: 7) + command.setBuffer(_xHat.metal, atIndex: 8) + + command.dispatchThreads( + width: _nbNeurons, + height: batchSize * sequence + ) + command.enqueue() + } + + /// Compute the averages of the different independent normalization units. + private func _computeμ(_ layer: LayerNormSeq) + { + let batchSize = layer.batchSize + let sequence = layer.sequence + + let pNbNeurons: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pSequence: [UInt32] = [UInt32(sequence)] + + if _μ == nil + { + _μ = MetalPrivateBuffer( + batchSize * sequence, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "computeLayerNormSeqμ", deviceID: _deviceID + ) + command.setBuffer(layer.outs.metal, atIndex: 0) + command.setBytes(pNbNeurons, atIndex: 1) + command.setBytes(pNbBatch, atIndex: 2) + command.setBytes(pSequence, atIndex: 3) + command.setBuffer(_μ.metal, atIndex: 4) + + command.dispatchThreads(width: sequence, height: batchSize) + command.enqueue() + } + + /// Compute the deviations of the different independent normalization units. + private func _computeσ2(_ layer: LayerNormSeq) + { + let batchSize = layer.batchSize + let sequence = layer.sequence + + let pNbNeurons: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pSequence: [UInt32] = [UInt32(sequence)] + + if _σ2 == nil + { + _σ2 = MetalPrivateBuffer( + batchSize * sequence, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "computeLayerNormSeqσ2", deviceID: _deviceID + ) + command.setBuffer(layer.outs.metal, atIndex: 0) + command.setBuffer(_μ.metal, atIndex: 1) + command.setBytes(pNbNeurons, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBytes(pSequence, atIndex: 4) + command.setBuffer(_σ2.metal, atIndex: 5) + + command.dispatchThreads(width: sequence, height: batchSize) + command.enqueue() + } + + /// Apply the backward pass in the GPU execution context. + func backward(_ layer: LayerNormSeq) + { + _backwardWeights1(layer) + _backwardWeights2(layer) + + let batchSize = layer.batchSize + let sequence = layer.sequence + + let pNbNeurons: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pSequence: [UInt32] = [UInt32(sequence)] + + let command = MetalKernel.get.createCommand( + "backwardLayerNormSeq", deviceID: _deviceID + ) + command.setBuffer(_σ2.metal, atIndex: 0) + command.setBuffer(_xHat.metal, atIndex: 1) + command.setBuffer(_Ɣ.w.metal, atIndex: 2) + command.setBuffer(_sum1.metal, atIndex: 3) + command.setBuffer(_sum2.metal, atIndex: 4) + command.setBytes(pNbNeurons, atIndex: 5) + command.setBytes(pNbBatch, atIndex: 6) + command.setBytes(pSequence, atIndex: 7) + command.setBuffer(layer.delta.metal, atIndex: 8) + + command.dispatchThreads( + width: _nbNeurons, + height: batchSize * sequence + ) + command.enqueue() + } + + /// Compute the gradients of weights in the GPU execution context. + private func _backwardWeights1(_ layer: LayerNormSeq) + { + let batchSize = layer.batchSize + let sequence = layer.sequence + + let pNbNeurons: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pSequence: [UInt32] = [UInt32(sequence)] + + if _sum1 == nil + { + _sum1 = MetalPrivateBuffer( + batchSize * sequence, deviceID: _deviceID + ) + _sum2 = MetalPrivateBuffer( + batchSize * sequence, deviceID: _deviceID + ) + } + + let command = MetalKernel.get.createCommand( + "backwardWeights1LayerNormSeq", deviceID: _deviceID + ) + command.setBuffer(layer.delta.metal, atIndex: 0) + command.setBuffer(_xHat.metal, atIndex: 1) + command.setBuffer(_Ɣ.w.metal, atIndex: 2) + command.setBytes(pNbNeurons, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBytes(pSequence, atIndex: 5) + command.setBuffer(_sum1.metal, atIndex: 6) + command.setBuffer(_sum2.metal, atIndex: 7) + + command.dispatchThreads(width: sequence, height: batchSize) + command.enqueue() + } + + /// Compute the gradients of weights in the GPU execution context. + private func _backwardWeights2(_ layer: LayerNormSeq) + { + let batchSize = layer.batchSize + let sequence = layer.sequence + + let pNbNeurons: [UInt32] = [UInt32(_nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pSequence: [UInt32] = [UInt32(sequence)] + let pAccumulate: [UInt32] = layer.accumulateDeltaWeights ? [1] : [0] + + let command = MetalKernel.get.createCommand( + "backwardWeights2LayerNormSeq", deviceID: _deviceID + ) + command.setBuffer(layer.delta.metal, atIndex: 0) + command.setBuffer(_xHat.metal, atIndex: 1) + command.setBytes(pNbNeurons, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBytes(pSequence, atIndex: 4) + command.setBytes(pAccumulate, atIndex: 5) + command.setBuffer(_Ɣ.g.metal, atIndex: 6) + command.setBuffer(_β.g.metal, atIndex: 7) + + command.dispatchThreads(_nbNeurons) + command.enqueue() + } + + /// Get the weights in the GPU execution context. + func collectWeights() -> [IWeightBuffers] + { + return [_Ɣ, _β] + } +} diff --git a/Sources/GrAIdient/Core/Layer/LayerUpdate.swift b/Sources/GrAIdient/Core/Layer/LayerUpdate.swift index 25e84981..6c6c31d3 100644 --- a/Sources/GrAIdient/Core/Layer/LayerUpdate.swift +++ b/Sources/GrAIdient/Core/Layer/LayerUpdate.swift @@ -7,12 +7,32 @@ import Foundation +/// Error occuring in an output layer. +public enum LossError: Error +{ + /// Ground truth has an unexected value. + case GroundTruthValue +} + +extension LossError: CustomStringConvertible +{ + public var description: String + { + switch self + { + case .GroundTruthValue: + return "Ground truth has an unexpected value." + } + } +} + /// Running phase of a model. public enum Phase { case Training, Inference } +/// API for a layer that have learning weights. public protocol LayerUpdate: Layer { /// Weights in the CPU execution context. @@ -197,3 +217,179 @@ class WeightBuffers: IWeightBuffers _vHat = nil } } + +/// Method used to initialize weights values (not considering the biases). +public enum WeightInitClass +{ + case XavierUniform, XavierNormal, KaimingUniform, KaimingNormal +} + +public protocol LayerWeightInit: LayerUpdate +{ + /// Number of weights values (not considering the biases). + var weightListSize: Int { get } + /// Method used to initialize weights values. + var weightInitClass: WeightInitClass { get set } + + /// Get the number of input and output connections. + var connectivityIO: (Int, Int) { get } + /// Get coefficient (depending on activation function) to apply during the weights initialization. + var coeffInitWeights: Float { get } +} + +extension LayerWeightInit +{ + /// Number of weights values. + public var weightListSize: Int + { + get { + let io = connectivityIO + return io.0 * io.1 + } + } + + /// Get coefficient (depending on activation function) to apply during the weights initialization. + public var coeffInitWeights: Float + { + get { + return 1.0 + } + } + + /// Generate list of weights values. + public func generateWeightsList() -> [Float] + { + let nbElems = weightListSize + let weightsList: [Float] + switch weightInitClass { + case .XavierUniform: + weightsList = Self.XavierUniform( + nbElems: nbElems, + connectivityIO: connectivityIO + ) + case .XavierNormal: + weightsList = Self.XavierNormal( + nbElems: nbElems, + connectivityIO: connectivityIO + ) + case .KaimingUniform: + weightsList = Self.KaimingUniform( + nbElems: nbElems, + coeff: coeffInitWeights, + connectivityIO: connectivityIO + ) + case .KaimingNormal: + weightsList = Self.KaimingNormal( + nbElems: nbElems, + coeff: coeffInitWeights, + connectivityIO: connectivityIO + ) + } + return weightsList + } + + /// + /// Xavier uniform initialization method. + /// + /// - Parameters: + /// - nbElems: Number of weights to initialize. + /// - connectivityIO: Number of input and output connections. + /// - Returns: Weights values. + /// + static func XavierUniform( + nbElems: Int, + connectivityIO: (Int, Int)) -> [Float] + { + var values = [Float]() + let bound = sqrt(6) / sqrt(Float(connectivityIO.0 + connectivityIO.1)) + for _ in 0.. [Float] + { + var values = [Float]() + let std = sqrt(2) / sqrt(Float(connectivityIO.0 + connectivityIO.1)) + for _ in 0.. [Float] + { + var values = [Float]() + let bound = sqrt(3) * coeff / sqrt(Float(connectivityIO.0)) + for _ in 0.. [Float] + { + var values = [Float]() + let std = coeff / sqrt(Float(connectivityIO.0)) + for _ in 0..( + mean: T, + standardDeviation: T) -> T +{ + let u1 = Double.random(in: 0..<1) + let u2 = Double.random(in: 0..<1) + let randStdNormal = sqrt(-2 * log(u1)) * cos(2 * .pi * u2) + let randNormal = mean + standardDeviation * T(randStdNormal) + return randNormal +} diff --git a/Sources/GrAIdient/Core/Model/Model.swift b/Sources/GrAIdient/Core/Model/Model.swift index 9a2843c4..0e603ac2 100644 --- a/Sources/GrAIdient/Core/Model/Model.swift +++ b/Sources/GrAIdient/Core/Model/Model.swift @@ -295,6 +295,33 @@ public class Model: BaseModel } } + /// The method used to initialize weights values (not considering the biases). + public var weightInitClass: WeightInitClass? + { + get { + for layer in layers + { + if let layerInit = layer as? LayerWeightInit + { + return layerInit.weightInitClass + } + } + return nil + } + set { + if let initClass = newValue + { + for layer in layers + { + if let layerInit = layer as? LayerWeightInit + { + layerInit.weightInitClass = initClass + } + } + } + } + } + /// Get/Set the weights of the different layers. public var weights: [[Float]] { diff --git a/Sources/GrAIdient/Layer1D/Activation1D.swift b/Sources/GrAIdient/Layer1D/Activation1D.swift index be710cb3..c4e8c590 100644 --- a/Sources/GrAIdient/Layer1D/Activation1D.swift +++ b/Sources/GrAIdient/Layer1D/Activation1D.swift @@ -18,6 +18,18 @@ public class Activation1D: Layer1D /// var _tmp: MetalPrivateBuffer! = nil + /// Get coefficient (depending on activation function) to apply during the weights initialization. + public var coeffInitWeights: Float + { + get { + if let activation = _activation + { + return activation.coeffInitWeights + } + return 1.0 + } + } + private enum Keys: String, CodingKey { case activation diff --git a/Sources/GrAIdient/Layer1D/BCE1D.swift b/Sources/GrAIdient/Layer1D/BCE1D.swift new file mode 100644 index 00000000..da842382 --- /dev/null +++ b/Sources/GrAIdient/Layer1D/BCE1D.swift @@ -0,0 +1,409 @@ +// +// BCE1D.swift +// GrAIdient +// +// Created by Jean-François Reboud on 05/07/2023. +// + +import Foundation + +/// Output layer with a 1D shape neural structure and a loss that computes binary cross entropy. +public class BCE1D: LayerOutput1D +{ + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer1D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = BCE1D(layerPrev: layerPrev, params: params) + layer.coeff = self.coeff + + return layer + } + + /// + /// Estimate the gradients of weights thanks to Gradient Checking. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// - Returns: The estimated gradients of weights. + /// + public func collectGradientsApprox( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> [T] + { + var gradients = [T]() + let nbGradients = neurons.get(0)!.nbGC / 2 + for elem in 0..( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int, + elem: Int) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + + var losses = [T](repeating: 0.0, count: batchSize) + for batch in 0..( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + + var losses = [T](repeating: 0.0, count: batchSize) + for elem in 0..( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> T + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + return try T(getLossGPU( + self.groundTruth, + batchSize: groundTruth.count, + nbNeurons: nbNeurons + )) + } + + /// + /// Get loss in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// - Returns: The loss value. + /// + public func getLossGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbNeurons: Int) throws -> Float + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + try checkLossGPU(batchSize: batchSize) + + let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + + let command = MetalKernel.get.createCommand( + "BCE1DLoss", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbNeurons, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBuffer(loss.metal, atIndex: 4) + + command.dispatchThreads(batchSize) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for i in 0..( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + + if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward + { + let neuronsPrev = layerPrev.neurons + for elem in 0..( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + try lossDerivativeGPU( + self.groundTruth, + batchSize: groundTruth.count, + nbNeurons: nbNeurons + ) + } + + /// + /// Compute the derivative of the loss in the GPU execution context. + /// + /// This function is necessary to initialize the backward pass ! + /// In a way, it plays a similar role as the `setData` of the first layer. + /// + /// The `setData` API sets data to the first layer to initialize the forward pass. + /// Here we use the `groundTruth` to initialize the backward pass. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// + public func lossDerivativeGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbNeurons: Int) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + + if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward + { + try layerPrev.checkStateBackwardGPU(batchSize: batchSize) + + let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] + let pCoeff: [Float] = [Float(coeff)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDirty: [UInt32] = layerPrev.dirty ? [1] : [0] + + let command = MetalKernel.get.createCommand( + "BCE1DLossDerivative", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbNeurons, atIndex: 2) + command.setBytes(pCoeff, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBytes(pDirty, atIndex: 5) + command.setBuffer(layerPrev.delta.metal, atIndex: 6) + + command.dispatchThreads( + width: nbNeurons, + height: batchSize + ) + command.enqueue() + + propagateDirty() + } + } +} diff --git a/Sources/GrAIdient/Layer1D/BCESigmoid1D.swift b/Sources/GrAIdient/Layer1D/BCESigmoid1D.swift new file mode 100644 index 00000000..237d3da3 --- /dev/null +++ b/Sources/GrAIdient/Layer1D/BCESigmoid1D.swift @@ -0,0 +1,428 @@ +// +// BCESigmoid1D.swift +// GrAIdient +// +// Created by Jean-François Reboud on 07/07/2023. +// + +import Foundation + +/// +/// Output layer with a 1D shape neural structure and a loss that computes binary cross entropy on top +/// of a sigmoid activation. +/// +public class BCESigmoid1D: LayerOutput1D +{ + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer1D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = BCESigmoid1D(layerPrev: layerPrev, params: params) + layer.coeff = self.coeff + + return layer + } + + /// + /// Estimate the gradients of weights thanks to Gradient Checking. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// - Returns: The estimated gradients of weights. + /// + public func collectGradientsApprox( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> [T] + { + var gradients = [T]() + let nbGradients = neurons.get(0)!.nbGC / 2 + for elem in 0..( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int, + elem: Int) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + + var losses = [T](repeating: 0.0, count: batchSize) + for batch in 0.. 0 + { + value = T(1 - gt[depth]) * T(out) + value += T(log(1 + exp(-out))) + } + else + { + value = -T(out) * T(gt[depth]) + value += T(log(exp(out) + 1)) + } + + losses[batch] += value + } + } + return T(coeff) * losses.reduce(0, +) / + T(nbNeurons * batchSize) + } + + /// + /// Get loss in the CPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// - Returns: The loss value. + /// + public func getLossCPU( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + + var losses = [T](repeating: 0.0, count: batchSize) + for elem in 0.. 0 + { + value = T(1 - gt[depth]) * T(out) + value += T(log(1 + exp(-out))) + } + else + { + value = -T(out) * T(gt[depth]) + value += T(log(exp(out) + 1)) + } + + losses[elem] += value + } + } + return T(coeff) * losses.reduce(0, +) / + T(nbNeurons * batchSize) + } + + /// + /// Get loss in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// - Returns: The loss value. + /// + public func getLossGPU( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> T + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + return try T(getLossGPU( + self.groundTruth, + batchSize: groundTruth.count, + nbNeurons: nbNeurons + )) + } + + /// + /// Get loss in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// - Returns: The loss value. + /// + public func getLossGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbNeurons: Int) throws -> Float + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + try checkLossGPU(batchSize: batchSize) + + let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + + let command = MetalKernel.get.createCommand( + "BCESigmoid1DLoss", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbNeurons, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBuffer(loss.metal, atIndex: 4) + + command.dispatchThreads(batchSize) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for i in 0..( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + + if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward + { + let neuronsPrev = layerPrev.neurons + for elem in 0..= 0 + { + value = 1.0 / (1.0 + exp(-out)) + } + else + { + value = exp(out) / (1.0 + exp(out)) + } + + if layerPrev.dirty + { + neuronsPrev.get(depth)!.v[elem].delta = + coeff * (value - Double(gt[depth])) / + Double(nbNeurons * batchSize) + } + else + { + neuronsPrev.get(depth)!.v[elem].delta += + coeff * (value - Double(gt[depth])) / + Double(nbNeurons * batchSize) + } + } + } + propagateDirty() + } + } + + /// + /// Compute the derivative of the loss in the GPU execution context. + /// + /// This function is necessary to initialize the backward pass ! + /// In a way, it plays a similar role as the `setData` of the first layer. + /// + /// The `setData` API sets data to the first layer to initialize the forward pass. + /// Here we use the `groundTruth` to initialize the backward pass. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// + public func lossDerivativeGPU( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + try lossDerivativeGPU( + self.groundTruth, + batchSize: groundTruth.count, + nbNeurons: nbNeurons + ) + } + + /// + /// Compute the derivative of the loss in the GPU execution context. + /// + /// This function is necessary to initialize the backward pass ! + /// In a way, it plays a similar role as the `setData` of the first layer. + /// + /// The `setData` API sets data to the first layer to initialize the forward pass. + /// Here we use the `groundTruth` to initialize the backward pass. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// + public func lossDerivativeGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbNeurons: Int) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + + if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward + { + try layerPrev.checkStateBackwardGPU(batchSize: batchSize) + + let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] + let pCoeff: [Float] = [Float(coeff)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDirty: [UInt32] = layerPrev.dirty ? [1] : [0] + + let command = MetalKernel.get.createCommand( + "BCESigmoid1DLossDerivative", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbNeurons, atIndex: 2) + command.setBytes(pCoeff, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBytes(pDirty, atIndex: 5) + command.setBuffer(layerPrev.delta.metal, atIndex: 6) + + command.dispatchThreads( + width: nbNeurons, + height: batchSize + ) + command.enqueue() + + propagateDirty() + } + } +} diff --git a/Sources/GrAIdient/Layer1D/Base/LayerInput1D.swift b/Sources/GrAIdient/Layer1D/Base/LayerInput1D.swift index d7de3105..d34977af 100644 --- a/Sources/GrAIdient/Layer1D/Base/LayerInput1D.swift +++ b/Sources/GrAIdient/Layer1D/Base/LayerInput1D.swift @@ -5,7 +5,7 @@ // Created by Jean-François Reboud on 09/10/2022. // -/// First layer of a model. +/// Input layer of a model. open class LayerInput1D: Layer1D { /// @@ -38,4 +38,113 @@ open class LayerInput1D: Layer1D computeDelta = true } } + + /// + /// Check and setup input in the CPU execution context. + /// + /// Throw an error if data size is not coherent. + /// + /// - Parameters: + /// - data: The input data. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// + public func checkInputCPU( + _ data: [[T]], + batchSize: Int, + nbNeurons: Int) throws + { + if data.count != batchSize || data.first!.count != nbNeurons + { + throw LayerError.DataSize + } + if nbNeurons != self.nbNeurons + { + throw LayerError.DataSize + } + try checkStateCPU(batchSize: batchSize) + + for (elem, sample) in data.enumerated() + { + if sample.count != nbNeurons + { + throw LayerError.DataSize + } + + for (i, feature) in sample.enumerated() { + if let neuron = neurons.get(i) + { + neuron.v[elem].out = Double(feature) + }} + } + } + + /// + /// Check and setup input in the GPU execution context. + /// + /// Throw an error if data size is not coherent. + /// + /// - Parameters: + /// - data: The input data. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// + public func checkInputGPU( + _ data: [[T]], + batchSize: Int, + nbNeurons: Int) throws + { + if data.count != batchSize || data.first!.count != nbNeurons + { + throw LayerError.DataSize + } + if nbNeurons != self.nbNeurons + { + throw LayerError.DataSize + } + try checkStateForwardGPU(batchSize: batchSize) + + // Wait for previous loop to end to avoid race condition with + // didModifyRange in the following example: + // FullyConnected.backwardWeightsGPU accesses layerPrev.outs. + MetalKernel.get.download([outs]) + + let outsPtr = outs.shared.buffer + for elem in 0.., + batchSize: Int, + nbNeurons: Int) throws + { + if data.nbElems > batchSize * nbNeurons + { + throw LayerError.DataSize + } + if nbNeurons != self.nbNeurons + { + throw LayerError.DataSize + } + try checkStateForwardGPU(batchSize: batchSize) + outs = data + } } diff --git a/Sources/GrAIdient/Layer1D/Base/LayerOutput1D.swift b/Sources/GrAIdient/Layer1D/Base/LayerOutput1D.swift index 6ee1d407..22200116 100644 --- a/Sources/GrAIdient/Layer1D/Base/LayerOutput1D.swift +++ b/Sources/GrAIdient/Layer1D/Base/LayerOutput1D.swift @@ -5,10 +5,10 @@ // Created by Jean-François Reboud on 09/10/2022. // -/// Last layer of a model. +/// Loss layer of a model with a 1D shape neural structure. open class LayerOutput1D: Layer1D { - /// Coefficient to be applied to the loss compuptation. + /// Coefficient to be applied to the loss computation. public var coeff: Double = 1.0 /// @@ -88,6 +88,137 @@ open class LayerOutput1D: Layer1D loss = nil } + /// + /// Check and setup ground truth in the CPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// + public func checkGroundTruthCPU( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws + { + if groundTruth.count != batchSize || + groundTruth.first!.count != nbNeurons + { + throw LayerError.DataSize + } + if batchSize != self.batchSize || + nbNeurons != self.nbNeurons + { + throw LayerError.DataSize + } + if batchSize <= 0 || batchSize > neurons.get(0)!.v.count + { + throw LayerError.BatchSize + } + } + + /// + /// Check and setup ground truth in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// + public func checkGroundTruthGPU( + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws + { + if groundTruth.count != batchSize || + groundTruth.first!.count != nbNeurons + { + throw LayerError.DataSize + } + if batchSize != self.batchSize || + nbNeurons != self.nbNeurons + { + throw LayerError.DataSize + } + + if self.groundTruth == nil + { + self.groundTruth = MetalSharedBuffer( + batchSize * nbNeurons, + deviceID: deviceID + ) + } + else if batchSize <= 0 || + batchSize * nbNeurons > self.groundTruth.nbElems + { + throw LayerError.BatchSize + } + + let bufferPtr = self.groundTruth.buffer + for (i, dataI) in groundTruth.enumerated() + { + if dataI.count != nbNeurons + { + throw LayerError.DataSize + } + for (j, dataIJ) in dataI.enumerated() + { + bufferPtr[j + i * nbNeurons] = Float(dataIJ) + } + } + MetalKernel.get.upload([self.groundTruth]) + } + + /// + /// Check and setup ground truth in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. + /// + public func checkGroundTruthGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbNeurons: Int) throws + { + if batchSize <= 0 || + batchSize * nbNeurons > groundTruth.nbElems + { + throw LayerError.BatchSize + } + if batchSize != self.batchSize || + nbNeurons != self.nbNeurons + { + throw LayerError.DataSize + } + } + + /// + /// Setup loss state in the GPU execution context. + /// + /// Throw an error if batch size or ground truth are incoherent. + /// + /// - Parameter batchSize: The batch size of data. + /// + public func checkLossGPU(batchSize: Int) throws + { + if loss == nil + { + loss = MetalSharedBuffer(batchSize, deviceID: deviceID) + } + else if batchSize > loss.nbElems + { + throw LayerError.BatchSize + } + } + /// /// Apply the forward pass of the Gradient Checking in CPU execution context. /// @@ -177,7 +308,7 @@ open class LayerOutput1D: Layer1D { // Note that backward is not called except when it is // an intermediate layer. - // Model.backward is only called on not dirty layers. + // Model.backward is only called on non dirty layers. if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward { let neuronsPrev = layerPrev.neurons @@ -207,7 +338,7 @@ open class LayerOutput1D: Layer1D { // Note that backward is not called except when it is // an intermediate layer. - // Model.backward is only called on not dirty layers. + // Model.backward is only called on non dirty layers. if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward { try layerPrev.checkStateBackwardGPU(batchSize: batchSize) diff --git a/Sources/GrAIdient/Layer1D/Concat1D.swift b/Sources/GrAIdient/Layer1D/Concat1D.swift index 8be8263b..f163a8d5 100644 --- a/Sources/GrAIdient/Layer1D/Concat1D.swift +++ b/Sources/GrAIdient/Layer1D/Concat1D.swift @@ -256,7 +256,7 @@ public class Concat1D: LayerMerge1D { try checkStateForwardGPU(batchSize: batchSize) - let pNbNeurones: [UInt32] = [UInt32(nbNeurons)] + let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] let pNbBatch: [UInt32] = [UInt32(batchSize)] let metalKernel = MetalKernel.get @@ -277,7 +277,7 @@ public class Concat1D: LayerMerge1D (_layersPrev[num] as! Layer1D).outs.metal, atIndex: 0 ) command.setBytes(pGlobalOffset, atIndex: 1) - command.setBytes(pNbNeurones, atIndex: 2) + command.setBytes(pNbNeurons, atIndex: 2) command.setBytes(pNbNeuronsPrev, atIndex: 3) command.setBytes(pNbBatch, atIndex: 4) command.setBuffer(outs.metal, atIndex: 5) @@ -347,7 +347,7 @@ public class Concat1D: LayerMerge1D return } - let pNbNeurones: [UInt32] = [UInt32(nbNeurons)] + let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] let pNbBatch: [UInt32] = [UInt32(batchSize)] let metalKernel = MetalKernel.get @@ -376,7 +376,7 @@ public class Concat1D: LayerMerge1D ) command.setBuffer(delta.metal, atIndex: 0) command.setBytes(pGlobalOffset, atIndex: 1) - command.setBytes(pNbNeurones, atIndex: 2) + command.setBytes(pNbNeurons, atIndex: 2) command.setBytes(pNbNeuronsPrev, atIndex: 3) command.setBytes(pNbBatch, atIndex: 4) command.setBytes(pDirty, atIndex: 5) diff --git a/Sources/GrAIdient/Layer1D/Constant1D.swift b/Sources/GrAIdient/Layer1D/Constant1D.swift index 1d73a75a..fd626737 100644 --- a/Sources/GrAIdient/Layer1D/Constant1D.swift +++ b/Sources/GrAIdient/Layer1D/Constant1D.swift @@ -5,8 +5,6 @@ // Created by Jean-François Reboud on 28/01/2023. // -import Foundation - /// Layer with a 1D shape neural structure and weights. public class Constant1D: Layer1D, LayerUpdate { @@ -316,13 +314,13 @@ public class Constant1D: Layer1D, LayerUpdate } for batch in 0.. Layer { let layerPrev = self.layerPrev! - let layer = FullyConnected( + let layer = try! FullyConnected( layerPrev: layerPrev, nbNeurons: nbNeurons, activation: nil, biases: _updateBiases, params: params ) + // Check fails if previousLayer is a Layer2D that is resized. + if weightWidth != layer.weightWidth + { + fatalError() + } + if GrAI.Opti.GPU { layer.weightsGPU = weightsGPU @@ -509,50 +525,39 @@ public class FullyConnected: Activation1D, LayerWithActivation, LayerUpdate /// public func initWeightsCPU() { + if _weightsList.count == 0 + { + _weightsList = generateWeightsList() + _weightsList += [Float](repeating: 0.0, count: weightHeight) + } + _wArrays = WeightGrids(width: weightWidth, height: weightHeight) _bArrays = WeightArrays(weightHeight) - if _weightsList.count == 0 + for i in 0.., IWeightBuffers } } -/// First layer with a 1D shape neural structure. +/// Input layer with a 1D shape neural structure. public class Input1D: LayerInput1D, LayerUpdate { /// Grid of "weights". @@ -262,26 +262,17 @@ public class Input1D: LayerInput1D, LayerUpdate /// /// Throw an error if data size is not coherent. /// - /// - Parameter data: The data to set. + /// - Parameters: + /// - data: The input data. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. /// - public func setDataCPU(_ data: [[T]]) throws + public func setDataCPU( + _ data: [[T]], + batchSize: Int, + nbNeurons: Int) throws { - let batchSize = data.count - try checkStateCPU(batchSize: batchSize) - - for (elem, sample) in data.enumerated() - { - if sample.count != nbNeurons - { - throw LayerError.DataSize - } - - for (i, feature) in sample.enumerated() { - if let neuron = neurons.get(i) - { - neuron.v[elem].out = Double(feature) - }} - } + try checkInputCPU(data, batchSize: batchSize, nbNeurons: nbNeurons) } /// @@ -289,33 +280,17 @@ public class Input1D: LayerInput1D, LayerUpdate /// /// Throw an error if data size is not coherent. /// - /// - Parameter data: The data to set. + /// - Parameters: + /// - data: The input data. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. /// - public func setDataGPU(_ data: [[T]]) throws + public func setDataGPU( + _ data: [[T]], + batchSize: Int, + nbNeurons: Int) throws { - let batchSize = data.count - try checkStateForwardGPU(batchSize: batchSize) - - if nbNeurons != data.first!.count - { - throw LayerError.DataSize - } - - // Wait for previous loop to end to avoid race condition with - // didModifyRange in the following example: - // FullyConnected.backwardWeightsGPU accesses layerPrev.outs. - MetalKernel.get.download([outs]) - - let outsPtr = outs.shared.buffer - for elem in 0.., - batchSize: Int) throws + batchSize: Int, + nbNeurons: Int) throws { - try checkStateForwardGPU(batchSize: batchSize) - outs = data + try checkInputGPU(data, batchSize: batchSize, nbNeurons: nbNeurons) } /// diff --git a/Sources/GrAIdient/Layer1D/LinearError1D.swift b/Sources/GrAIdient/Layer1D/LinearError1D.swift index 90ad453c..6549eeea 100644 --- a/Sources/GrAIdient/Layer1D/LinearError1D.swift +++ b/Sources/GrAIdient/Layer1D/LinearError1D.swift @@ -5,7 +5,7 @@ // Created by Jean-François Reboud on 10/10/2022. // -/// Last layer with a 1D shape neural structure and a loss function that depends linearly on its inputs. +/// Output layer with a 1D shape neural structure and a loss function that depends linearly on its inputs. public class LinearError1D: LayerOutput1D { /// @@ -39,20 +39,35 @@ public class LinearError1D: LayerOutput1D /// /// Estimate the gradients of weights thanks to Gradient Checking. /// - /// Throw an error if batch size or ground truth are incoherent. + /// Throw an error if data size is incoherent. /// - /// - Parameter groundTruth: The ground truth. + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. /// - Returns: The estimated gradients of weights. /// public func collectGradientsApprox( - _ groundTruth: [[T]]) throws -> [T] + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> [T] { var gradients = [T]() let nbGradients = neurons.get(0)!.nbGC / 2 for elem in 0..( _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int, elem: Int) throws -> T { - let batchSize = groundTruth.count - if batchSize != self.batchSize || - batchSize <= 0 || batchSize > neurons.get(0)!.v.count - { - throw LayerError.BatchSize - } + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) var losses = [T](repeating: 0.0, count: batchSize) for batch in 0..( - _ groundTruth: [[T]]) throws -> T + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> T { - let batchSize = groundTruth.count - if batchSize != self.batchSize || - batchSize <= 0 || batchSize > neurons.get(0)!.v.count - { - throw LayerError.BatchSize - } + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) var losses = [T](repeating: 0.0, count: batchSize) for elem in 0..( - _ groundTruth: [[T]]) throws -> T + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> T { - let batchSize = groundTruth.count - if self.groundTruth == nil - { - self.groundTruth = MetalSharedBuffer( - batchSize * nbNeurons, - deviceID: deviceID - ) - } - - let bufferPtr = self.groundTruth.buffer - for (i, dataI) in groundTruth.enumerated() - { - if dataI.count != nbNeurons - { - throw LayerError.DataSize - } - for (j, dataIJ) in dataI.enumerated() - { - bufferPtr[j + i * nbNeurons] = Float(dataIJ) - } - } - MetalKernel.get.upload([self.groundTruth]) - + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) return try T(getLossGPU( self.groundTruth, batchSize: groundTruth.count @@ -195,27 +204,15 @@ public class LinearError1D: LayerOutput1D _ groundTruth: MetalBuffer, batchSize: Int) throws -> Float { + try checkLossGPU(batchSize: batchSize) if batchSize != self.batchSize { throw LayerError.BatchSize } - if batchSize * nbNeurons > groundTruth.nbElems - { - throw LayerError.DataSize - } let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] let pNbBatch: [UInt32] = [UInt32(batchSize)] - if loss == nil - { - loss = MetalSharedBuffer(batchSize, deviceID: deviceID) - } - if batchSize > loss.nbElems - { - throw LayerError.BatchSize - } - let command = MetalKernel.get.createCommand( "linearErrorLoss", deviceID: deviceID ) @@ -256,15 +253,23 @@ public class LinearError1D: LayerOutput1D throw LayerError.BatchSize } - if let layerPrev = self.layerPrev as? Layer1D + if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward { let neuronsPrev = layerPrev.neurons for elem in 0..( - batchSize * nbNeurons, deviceID: deviceID) - } - if batchSize * nbNeurons > layerPrev.delta.nbElems - { - throw LayerError.BatchSize - } + let pDirty: [UInt32] = layerPrev.dirty ? [1] : [0] let command = MetalKernel.get.createCommand( "linearErrorLossDerivative", deviceID: deviceID @@ -307,7 +305,8 @@ public class LinearError1D: LayerOutput1D command.setBytes(pNbNeurons, atIndex: 1) command.setBytes(pCoeff, atIndex: 2) command.setBytes(pNbBatch, atIndex: 3) - command.setBuffer(layerPrev.delta.metal, atIndex: 4) + command.setBytes(pDirty, atIndex: 4) + command.setBuffer(layerPrev.delta.metal, atIndex: 5) command.dispatchThreads( width: nbNeurons, diff --git a/Sources/GrAIdient/Layer1D/MSE1D.swift b/Sources/GrAIdient/Layer1D/MSE1D.swift index 15aa3cba..baeab33f 100644 --- a/Sources/GrAIdient/Layer1D/MSE1D.swift +++ b/Sources/GrAIdient/Layer1D/MSE1D.swift @@ -5,7 +5,7 @@ // Created by Jean-François Reboud on 10/10/2022. // -/// Last layer with a 1D shape neural structure and a loss that computes mean squared error. +/// Output layer with a 1D shape neural structure and a loss that computes mean squared error. public class MSE1D: LayerOutput1D { /// @@ -39,20 +39,35 @@ public class MSE1D: LayerOutput1D /// /// Estimate the gradients of weights thanks to Gradient Checking. /// - /// Throw an error if batch size or ground truth are incoherent. + /// Throw an error if data size is incoherent. /// - /// - Parameter groundTruth: The ground truth. + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. /// - Returns: The estimated gradients of weights. /// public func collectGradientsApprox( - _ groundTruth: [[T]]) throws -> [T] + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> [T] { var gradients = [T]() let nbGradients = neurons.get(0)!.nbGC / 2 for elem in 0..( _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int, elem: Int) throws -> T { - let batchSize = groundTruth.count - if batchSize <= 0 || batchSize > neurons.get(0)!.v.count - { - throw LayerError.BatchSize - } + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) var losses = [T](repeating: 0.0, count: batchSize) for batch in 0..( - _ groundTruth: [[T]]) throws -> T + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> T { - let batchSize = groundTruth.count - if batchSize != self.batchSize || - batchSize <= 0 || batchSize > neurons.get(0)!.v.count - { - throw LayerError.BatchSize - } + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) var losses = [T](repeating: 0.0, count: batchSize) for elem in 0..( - _ groundTruth: [[T]]) throws -> T + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws -> T { - let batchSize = groundTruth.count - if self.groundTruth == nil - { - self.groundTruth = MetalSharedBuffer( - batchSize * nbNeurons, - deviceID: deviceID - ) - } - - let bufferPtr = self.groundTruth.buffer - for (i, dataI) in groundTruth.enumerated() - { - if dataI.count != nbNeurons - { - throw LayerError.DataSize - } - for (j, dataIJ) in dataI.enumerated() - { - bufferPtr[j + i * nbNeurons] = Float(dataIJ) - } - } - MetalKernel.get.upload([self.groundTruth]) - + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) return try T(getLossGPU( self.groundTruth, - batchSize: groundTruth.count + batchSize: batchSize, + nbNeurons: nbNeurons )) } /// /// Get loss in the GPU execution context. /// - /// Throw an error if batch size or ground truth are incoherent. + /// Throw an error if data size is incoherent. /// /// - Parameters: /// - groundTruth: The ground truth. /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. /// - Returns: The loss value. /// public func getLossGPU( _ groundTruth: MetalBuffer, - batchSize: Int) throws -> Float + batchSize: Int, + nbNeurons: Int) throws -> Float { - if batchSize != self.batchSize - { - throw LayerError.BatchSize - } - if batchSize * nbNeurons > groundTruth.nbElems - { - throw LayerError.DataSize - } + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) + try checkLossGPU(batchSize: batchSize) let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] let pNbBatch: [UInt32] = [UInt32(batchSize)] - if loss == nil - { - loss = MetalSharedBuffer(batchSize, deviceID: deviceID) - } - if batchSize > loss.nbElems - { - throw LayerError.BatchSize - } - let command = MetalKernel.get.createCommand( "MSE1DLoss", deviceID: deviceID ) @@ -246,21 +248,25 @@ public class MSE1D: LayerOutput1D /// The `setData` API sets data to the first layer to initialize the forward pass. /// Here we use the `groundTruth` to initialize the backward pass. /// - /// Throw an error if batch size or ground truth are incoherent. + /// Throw an error if data size is incoherent. /// - /// - Parameter groundTruth: The ground truth. + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. /// public func lossDerivativeCPU( - _ groundTruth: [[T]]) throws + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws { - let batchSize = groundTruth.count - if batchSize != self.batchSize || - batchSize <= 0 || batchSize > neurons.get(0)!.v.count - { - throw LayerError.BatchSize - } + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) - if let layerPrev = self.layerPrev as? Layer1D + if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward { let neuronsPrev = layerPrev.neurons for elem in 0..( - _ groundTruth: [[T]]) throws + _ groundTruth: [[T]], + batchSize: Int, + nbNeurons: Int) throws { - let batchSize = groundTruth.count - if self.groundTruth == nil - { - self.groundTruth = MetalSharedBuffer( - batchSize * nbNeurons, - deviceID: deviceID - ) - } - - let bufferPtr = self.groundTruth.buffer - for (i, dataI) in groundTruth.enumerated() - { - if dataI.count != nbNeurons - { - throw LayerError.DataSize - } - for (j, dataIJ) in dataI.enumerated() - { - bufferPtr[j + i * nbNeurons] = Float(dataIJ) - } - } - MetalKernel.get.upload([self.groundTruth]) - + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) try lossDerivativeGPU( self.groundTruth, - batchSize: groundTruth.count + batchSize: groundTruth.count, + nbNeurons: nbNeurons ) } @@ -338,40 +342,32 @@ public class MSE1D: LayerOutput1D /// The `setData` API sets data to the first layer to initialize the forward pass. /// Here we use the `groundTruth` to initialize the backward pass. /// - /// Throw an error if batch size or ground truth are incoherent. + /// Throw an error if data size is incoherent. /// /// - Parameters: /// - groundTruth: The ground truth. /// - batchSize: The batch size of data. + /// - nbNeurons: Number of neurons. /// public func lossDerivativeGPU( _ groundTruth: MetalBuffer, - batchSize: Int) throws + batchSize: Int, + nbNeurons: Int) throws { - if batchSize != self.batchSize - { - throw LayerError.BatchSize - } - if batchSize * nbNeurons > groundTruth.nbElems - { - throw LayerError.DataSize - } + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbNeurons: nbNeurons + ) - if let layerPrev = self.layerPrev as? Layer1D + if let layerPrev = self.layerPrev as? Layer1D, mustComputeBackward { + try layerPrev.checkStateBackwardGPU(batchSize: batchSize) + let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] let pCoeff: [Float] = [Float(coeff)] let pNbBatch: [UInt32] = [UInt32(batchSize)] - - if layerPrev.delta == nil - { - layerPrev.delta = MetalPrivateBuffer( - batchSize * nbNeurons, deviceID: deviceID) - } - if batchSize * nbNeurons > layerPrev.delta.nbElems - { - throw LayerError.BatchSize - } + let pDirty: [UInt32] = layerPrev.dirty ? [1] : [0] let command = MetalKernel.get.createCommand( "MSE1DLossDerivative", deviceID: deviceID @@ -381,7 +377,8 @@ public class MSE1D: LayerOutput1D command.setBytes(pNbNeurons, atIndex: 2) command.setBytes(pCoeff, atIndex: 3) command.setBytes(pNbBatch, atIndex: 4) - command.setBuffer(layerPrev.delta.metal, atIndex: 5) + command.setBytes(pDirty, atIndex: 5) + command.setBuffer(layerPrev.delta.metal, atIndex: 6) command.dispatchThreads( width: nbNeurons, diff --git a/Sources/GrAIdient/Layer1D/Softmax1D.swift b/Sources/GrAIdient/Layer1D/Softmax1D.swift index fd71e081..c9e773f6 100644 --- a/Sources/GrAIdient/Layer1D/Softmax1D.swift +++ b/Sources/GrAIdient/Layer1D/Softmax1D.swift @@ -14,11 +14,11 @@ import Foundation /// public class Softmax1D: Layer1D { - let _size: Int + let _nbHeads: Int private enum Keys: String, CodingKey { - case size + case nbHeads } /// @@ -26,21 +26,23 @@ public class Softmax1D: Layer1D /// /// - Parameters: /// - layerPrev: Previous layer that has been queued to the model. - /// - size: The number of neurons per block. + /// - nbHeads: Number of heads (groups) of neurons. /// - params: Contextual parameters linking to the model. /// - public init(layerPrev: Layer1D, size: Int, params: GrAI.Model.Params) + public init(layerPrev: Layer1D, + nbHeads: Int, + params: GrAI.Model.Params) throws { let nbNeurons = layerPrev.nbNeurons - if nbNeurons % size != 0 + if nbNeurons % nbHeads != 0 { - fatalError( - "'nbNeurons' (\(nbNeurons) " + - "should be a multiple of size (\(size))." + throw LayerError.Init( + message: "`nbNeurons` (\(nbNeurons) " + + "should be a multiple of nbHeads (\(nbHeads))." ) } - _size = size + _nbHeads = nbHeads super.init(layerPrev: layerPrev, nbNeurons: nbNeurons, params: params) @@ -57,7 +59,7 @@ public class Softmax1D: Layer1D public required init(from decoder: Decoder) throws { let values = try decoder.container(keyedBy: Keys.self) - _size = try values.decode(Int.self, forKey: Keys.size) + _nbHeads = try values.decode(Int.self, forKey: Keys.nbHeads) try super.init(from: decoder) } @@ -75,7 +77,7 @@ public class Softmax1D: Layer1D public override func encode(to encoder: Encoder) throws { var container = encoder.container(keyedBy: Keys.self) - try container.encode(_size, forKey: Keys.size) + try container.encode(_nbHeads, forKey: Keys.nbHeads) try super.encode(to: encoder) } @@ -101,9 +103,9 @@ public class Softmax1D: Layer1D let params = GrAI.Model.Params(context: context) params.context.curID = id - let layer = Softmax1D( + let layer = try! Softmax1D( layerPrev: layerPrev, - size: _size, + nbHeads: _nbHeads, params: params ) return layer @@ -126,28 +128,40 @@ public class Softmax1D: Layer1D neurons.get(j)!.initGC(batchSize: batchSize, nbGC: nbGC) } - let nbBlocks = nbNeurons / _size + let size = nbNeurons / _nbHeads let neuronsPrev = layerPrev.neurons for batch in 0.. cMax + { + cMax = outPrev + } + } + var sum1 = 0.0 - for j1 in 0..<_size + for j in 0.. cMax { - let outPrev = neuronsPrev.get( - j1 + block * _size)!.v[elem].out - neurons.get(j1 + block * _size)!.v[elem].out = - exp(outPrev) / sum1 + cMax = outPrev } } - } + + var sum1 = 0.0 + for j in 0..! = nil + /// Get coefficient (depending on activation function) to apply during the weights initialization. + public var coeffInitWeights: Float + { + get { + if let activation = _activation + { + return activation.coeffInitWeights + } + return 1.0 + } + } + private enum Keys: String, CodingKey { case activation diff --git a/Sources/GrAIdient/Layer2D/AdaIN.swift b/Sources/GrAIdient/Layer2D/AdaIN.swift new file mode 100644 index 00000000..2fd50d6c --- /dev/null +++ b/Sources/GrAIdient/Layer2D/AdaIN.swift @@ -0,0 +1,785 @@ +// +// AdaIN.swift +// GrAIdient +// +// Created by Jean-François Reboud on 18/02/2023. +// + +/// +/// Layer with a 2D shape neural structure. +/// +/// This layer merges multiple (2) layers together, using style of second layer into the first one. +/// +public class AdaIN: LayerMerge2D +{ + /// Instance normalization by default or instance normalization in the CPU execution context. + var _norm: LayerWeightsNormalization? = nil + /// Instance normalization in the GPU execution context. + var _normGPU: InstanceNormalizationGPU? = nil + + /// Get instance normalization in the CPU execution context. + var norm: InstanceNormalization? + { + get { + return _norm as? InstanceNormalization + } + } + + private enum Keys: String, CodingKey + { + case norm + } + + /// Whether to compute gradients of previous main layer or not. + var computeDeltaMain: Bool + { + get { + let layerFirst = _layersPrev.first as! Layer2D + return layerFirst.computeDelta + } + } + /// Whether to compute gradients of previous style layer or not. + var computeDeltaStyle: Bool + { + get { + let layerLast = _layersPrev.last as! Layer1D + return layerLast.computeDelta + } + } + + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layersPrev: List of previous layers that have been queued to the model. + /// - params: Contextual parameters linking to the model. + /// + public init(layersPrev: [Layer], params: GrAI.Model.Params) throws + { + if layersPrev.count != 2 || + (layersPrev.first as? Layer2D) == nil || + (layersPrev.last as? Layer1D) == nil + { + throw LayerError.Init(message: "Layer structure error.") + } + + let layerFirst = layersPrev.first as! Layer2D + let layerLast = layersPrev.last as! Layer1D + + if layerLast.nbNeurons != 2 * layerFirst.nbChannels + { + throw LayerError.Init(message: "Layer structure error.") + } + + super.init(layersPrev: layersPrev, + nbChannels: layerFirst.nbChannels, + height: layerFirst.height, + width: layerFirst.width, + params: params) + + _norm = LayerWeightsNormalization(self) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let values = try decoder.container(keyedBy: Keys.self) + _norm = try values.decodeIfPresent( + LayerWeightsNormalization.self, forKey: .norm + ) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + if let norm = _normGPU + { + try container.encode(norm, forKey: Keys.norm) + } + else if let norm = _norm + { + try container.encode(norm, forKey: Keys.norm) + } + try super.encode(to: encoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + var layersPrev = [Layer]() + for idPrev in _idsPrev + { + layersPrev.append(mapping[idPrev]!) + } + + let layer = try! AdaIN(layersPrev: layersPrev, params: params) + if inPlace + { + layer._norm = _norm + layer._normGPU = _normGPU + } + else + { + // only one of them should be cloned + if let norm = _normGPU + { + layer._norm = norm.clone() + } + else if let norm = _norm + { + layer._norm = norm.clone() + } + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We reset batch normalization. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + norm?.resetKernel() + } + /// + /// Clean state resources in the GPU execution context. + /// + /// We reset batch normalization. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + _normGPU?.resetKernel() + } + + /// + /// Initialize hard resources in the CPU execution context. + /// + /// We initialize batch normalization. + /// + public override func initKernelCPU() + { + super.initKernelCPU() + + if let norm = _normGPU + { + _norm = InstanceNormalization(norm: norm) + } + else if let norm = _norm + { + _norm = InstanceNormalization(norm: norm) + } + + if !GrAI.Loop.gradientChecking + { + _normGPU = nil + } + } + + /// + /// Initialize hard resources in the GPU execution context. + /// + /// We initialize batch normalization. + /// + public override func initKernelGPU() + { + super.initKernelGPU() + + if let norm = _normGPU + { + _normGPU = InstanceNormalizationGPU(norm: norm) + } + else if let norm = _norm + { + _normGPU = InstanceNormalizationGPU(norm: norm) + } + _normGPU?.initKernel(deviceID: deviceID) + + if !GrAI.Loop.gradientChecking + { + _norm = nil + } + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: batchSize) + + let (nbSameElems, layersIndex, nbElems) = getMergedGraph() + + var nbGC = nbSameElems + for nbElemsTmp in nbElems + { + nbGC += nbElemsTmp + } + + for depth in 0.. [Double] + { + let layerFirst = _layersPrev.first as! Layer2D + var outs = [Double](repeating: 0.0, count: height * width) + + for i in 0.. Double + { + let layerLast = _layersPrev.last as! Layer1D + return layerLast.neurons.get(depth)!.gc[batch][elem].out + } + + /// + /// Set the outputs of Gradient Checking (result of the forward pass) in the CPU execution context. + /// + /// - Parameters: + /// - depth: Channel index. + /// - batch: Index sample in the mini batch. + /// - elem: Weight estimation index during the Gradient Checking. + /// - outs: The outputs to set. + /// + func setOutsGC(depth: Int, batch: Int, elem: Int, outs: [Double]) + { + for i in 0.. [Double] + { + let layerFirst = _layersPrev.first as! Layer2D + var outs = [Double](repeating: 0.0, count: height * width) + + for i in 0.. Double + { + let layerLast = _layersPrev.last as! Layer1D + return layerLast.neurons.get(depth)!.v[batch].out + } + + /// + /// Set the outputs (result of the forward pass) in the CPU execution context. + /// + /// - Parameters: + /// - depth: Channel index. + /// - batch: Index sample in the mini batch. + /// - outs: The outputs to set. + /// + func setOuts(depth: Int, batch: Int, outs: [Double]) + { + for i in 0.., + depth: Int, + batch: Int) -> [Double] + { + var outs = [Double](repeating: 0.0, count: height * width) + let offsetStart = (depth + nbChannels * batch) * height + + for i in 0.., + depth: Int, + batch: Int) -> Double + { + let layerLast = _layersPrev.last as! Layer1D + let offset = depth + layerLast.nbNeurons * batch + return Double(buffer[offset]) + } + + /// + /// Get the gradients (result of the backward pass) in the CPU execution context. + /// + /// - Parameters: + /// - depth: Channel index. + /// - batch: Index sample in the mini batch. + /// - Returns: The gradients. + /// + func getDelta(depth: Int, batch: Int) -> [Double] + { + var delta = [Double](repeating: 0.0, count: height * width) + for i in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = try! BCE2D(layerPrev: layerPrev, params: params) + layer.coeff = self.coeff + + return layer + } + + /// + /// Estimate the gradients of weights thanks to Gradient Checking. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: TThe data format. + /// - Returns: The estimated gradients of weights. + /// + public func collectGradientsApprox( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> [T] + { + var gradients = [T]() + let nbGradients = neurons.first!.get(0)!.nbGC / 2 + for elem in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + elem: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + var losses = [T](repeating: 0.0, count: batchSize) + switch format + { + case .RGB: + for batch in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + var losses = [T](repeating: 0.0, count: batchSize) + switch format + { + case .RGB: + for elem in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width, + format: format + ) + return try T(getLossGPU( + self.groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + )) + } + + /// + /// Get loss in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - Returns: The loss value. + /// + public func getLossGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws -> Float + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + try checkLossGPU(batchSize: batchSize) + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + + let command = MetalKernel.get.createCommand( + "BCE2DLoss", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBuffer(loss.metal, atIndex: 5) + + command.dispatchThreads(batchSize) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for i in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + if let layerPrev = self.layerPrev as? Layer2D, mustComputeBackward + { + let neuronsPrev = layerPrev.neurons + switch format + { + case .RGB: + for elem in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width, + format: format + ) + try lossDerivativeGPU( + self.groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + } + + /// + /// Compute the derivative of the loss in the GPU execution context. + /// + /// This function is necessary to initialize the backward pass ! + /// In a way, it plays a similar role as the `setData` of the first layer. + /// + /// The `setData` API sets data to the first layer to initialize the forward pass. + /// Here we use the `groundTruth` to initialize the backward pass. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// + public func lossDerivativeGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + if let layerPrev = self.layerPrev as? Layer2D, mustComputeBackward + { + try layerPrev.checkStateBackwardGPU(batchSize: batchSize) + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pCoeff: [Float] = [Float(coeff)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDirty: [UInt32] = layerPrev.dirty ? [1] : [0] + + let command = MetalKernel.get.createCommand( + "BCE2DLossDerivative", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBytes(pCoeff, atIndex: 4) + command.setBytes(pNbBatch, atIndex: 5) + command.setBytes(pDirty, atIndex: 6) + command.setBuffer(layerPrev.delta.metal, atIndex: 7) + + command.dispatchThreads( + width: nbChannels * width, + height: batchSize * height + ) + command.enqueue() + + propagateDirty() + } + } +} diff --git a/Sources/GrAIdient/Layer2D/BCESigmoid2D.swift b/Sources/GrAIdient/Layer2D/BCESigmoid2D.swift new file mode 100644 index 00000000..d1104542 --- /dev/null +++ b/Sources/GrAIdient/Layer2D/BCESigmoid2D.swift @@ -0,0 +1,570 @@ +// +// BCESigmoid2D.swift +// GrAIdient +// +// Created by Jean-François Reboud on 07/07/2023. +// + +import Foundation + +/// +/// Output layer with a 2D shape neural structure and a loss that computes binary cross entropy on top +/// of a sigmoid activation. +/// +public class BCESigmoid2D: LayerOutput2D +{ + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = try! BCESigmoid2D(layerPrev: layerPrev, params: params) + layer.coeff = self.coeff + + return layer + } + + /// + /// Estimate the gradients of weights thanks to Gradient Checking. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: TThe data format. + /// - Returns: The estimated gradients of weights. + /// + public func collectGradientsApprox( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> [T] + { + var gradients = [T]() + let nbGradients = neurons.first!.get(0)!.nbGC / 2 + for elem in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + elem: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + var losses = [T](repeating: 0.0, count: batchSize) + switch format + { + case .RGB: + for batch in 0.. 0 + { + value = T(1 - gt) * T(out) + value += T(log(1 + exp(-out))) + } + else + { + value = -T(out) * T(gt) + value += T(log(exp(out) + 1)) + } + + losses[batch] += value + }} + }} + case .Neuron: + for batch in 0.. 0 + { + value = T(1 - gt) * T(out) + value += T(log(1 + exp(-out))) + } + else + { + value = -T(out) * T(gt) + value += T(log(exp(out) + 1)) + } + + losses[batch] += value + }} + }} + } + return T(coeff) * losses.reduce(0, +) / + T(batchSize * nbChannels * height * width) + } + + /// + /// Get loss in the CPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: The data format. + /// - Returns: The loss value. + /// + public func getLossCPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + var losses = [T](repeating: 0.0, count: batchSize) + switch format + { + case .RGB: + for elem in 0.. 0 + { + value = T(1 - gt) * T(out) + value += T(log(1 + exp(-out))) + } + else + { + value = -T(out) * T(gt) + value += T(log(exp(out) + 1)) + } + + losses[elem] += value + }} + }} + case .Neuron: + for elem in 0.. 0 + { + value = T(1 - gt) * T(out) + value += T(log(1 + exp(-out))) + } + else + { + value = -T(out) * T(gt) + value += T(log(exp(out) + 1)) + } + + losses[elem] += value + }} + }} + } + return T(coeff) * losses.reduce(0, +) / + T(batchSize * nbChannels * height * width) + } + + /// + /// Get loss in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: The data format. + /// - Returns: The loss value. + /// + public func getLossGPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width, + format: format + ) + return try T(getLossGPU( + self.groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + )) + } + + /// + /// Get loss in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - Returns: The loss value. + /// + public func getLossGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws -> Float + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + try checkLossGPU(batchSize: batchSize) + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + + let command = MetalKernel.get.createCommand( + "BCESigmoid2DLoss", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBuffer(loss.metal, atIndex: 5) + + command.dispatchThreads(batchSize) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for i in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + if let layerPrev = self.layerPrev as? Layer2D, mustComputeBackward + { + let neuronsPrev = layerPrev.neurons + switch format + { + case .RGB: + for elem in 0..= 0 + { + value = 1.0 / (1.0 + exp(-out)) + } + else + { + value = exp(out) / (1.0 + exp(out)) + } + + if layerPrev.dirty + { + neuronsPrev[depth].get(i, j)!.v[elem].delta = + coeff * (value - Double(gt)) / + Double(batchSize * nbChannels * height * width) + } + else + { + neuronsPrev[depth].get(i, j)!.v[elem].delta += + coeff * (value - Double(gt)) / + Double(batchSize * nbChannels * height * width) + } + }} + }} + case .Neuron: + for elem in 0..= 0 + { + value = 1.0 / (1.0 + exp(-out)) + } + else + { + value = exp(out) / (1.0 + exp(out)) + } + + if layerPrev.dirty + { + neuronsPrev[depth].get(i, j)!.v[elem].delta = + coeff * (value - Double(gt)) / + Double(batchSize * nbChannels * height * width) + } + else + { + neuronsPrev[depth].get(i, j)!.v[elem].delta += + coeff * (value - Double(gt)) / + Double(batchSize * nbChannels * height * width) + } + }} + }} + } + propagateDirty() + } + } + + /// + /// Compute the derivative of the loss in the GPU execution context. + /// + /// This function is necessary to initialize the backward pass ! + /// In a way, it plays a similar role as the `setData` of the first layer. + /// + /// The `setData` API sets data to the first layer to initialize the forward pass. + /// Here we use the `groundTruth` to initialize the backward pass. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: The data format. + /// + public func lossDerivativeGPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width, + format: format + ) + try lossDerivativeGPU( + self.groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + } + + /// + /// Compute the derivative of the loss in the GPU execution context. + /// + /// This function is necessary to initialize the backward pass ! + /// In a way, it plays a similar role as the `setData` of the first layer. + /// + /// The `setData` API sets data to the first layer to initialize the forward pass. + /// Here we use the `groundTruth` to initialize the backward pass. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// + public func lossDerivativeGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + if let layerPrev = self.layerPrev as? Layer2D, mustComputeBackward + { + try layerPrev.checkStateBackwardGPU(batchSize: batchSize) + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pCoeff: [Float] = [Float(coeff)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDirty: [UInt32] = layerPrev.dirty ? [1] : [0] + + let command = MetalKernel.get.createCommand( + "BCESigmoid2DLossDerivative", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBytes(pCoeff, atIndex: 4) + command.setBytes(pNbBatch, atIndex: 5) + command.setBytes(pDirty, atIndex: 6) + command.setBuffer(layerPrev.delta.metal, atIndex: 7) + + command.dispatchThreads( + width: nbChannels * width, + height: batchSize * height + ) + command.enqueue() + + propagateDirty() + } + } +} diff --git a/Sources/GrAIdient/Layer2D/BN2D.swift b/Sources/GrAIdient/Layer2D/BN2D.swift index 12bc8a6b..17254239 100644 --- a/Sources/GrAIdient/Layer2D/BN2D.swift +++ b/Sources/GrAIdient/Layer2D/BN2D.swift @@ -9,9 +9,9 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation { /// Batch normalization by default or batch normalization in the CPU execution context. - var _bn: BatchNormalizationBase? = nil + var _norm: LayerWeightsStatsNormalization? = nil /// Batch normalization in the GPU execution context. - var _bnGPU: BatchNormalizationGPU? = nil + var _normGPU: BatchNormalizationGPU? = nil /// Whether to compute weights' gradients or not. public var computeDeltaWeights: Bool = true @@ -24,16 +24,16 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation { get { var weightsTmp = [Float]() - if let bn = _bn + if let norm = _norm { - weightsTmp += bn.weights + weightsTmp += norm.weights } return weightsTmp } set { - if let bn = _bn + if let norm = _norm { - bn.weights = newValue + norm.weights = newValue } } } @@ -43,24 +43,24 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation { get { var weightsTmp = [Float]() - if let bn = _bnGPU + if let norm = _normGPU { - weightsTmp += bn.weights + weightsTmp += norm.weights } - else if let bn = _bn + else if let norm = _norm { - weightsTmp += bn.weights + weightsTmp += norm.weights } return weightsTmp } set { - if let bn = _bnGPU + if let norm = _normGPU { - bn.weights = newValue + norm.weights = newValue } - else if let bn = _bn + else if let norm = _norm { - bn.weights = newValue + norm.weights = newValue } } } @@ -70,16 +70,16 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation { get { var statsTmp = [Float]() - if let bn = _bn + if let norm = _norm { - statsTmp += bn.stats + statsTmp += norm.stats } return statsTmp } set { - if let bn = _bn + if let norm = _norm { - bn.stats = newValue + norm.stats = newValue } } } @@ -89,33 +89,33 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation { get { var statsTmp = [Float]() - if let bn = _bnGPU + if let norm = _normGPU { - statsTmp += bn.stats + statsTmp += norm.stats } - else if let bn = _bn + else if let norm = _norm { - statsTmp += bn.stats + statsTmp += norm.stats } return statsTmp } set { - if let bn = _bnGPU + if let norm = _normGPU { - bn.stats = newValue + norm.stats = newValue } - else if let bn = _bn + else if let norm = _norm { - bn.stats = newValue + norm.stats = newValue } } } /// Get batch normalization in the CPU execution context. - var bn: BatchNormalization? + var norm: BatchNormalization? { get { - return _bn as? BatchNormalization + return _norm as? BatchNormalization } } @@ -129,7 +129,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation private enum Keys: String, CodingKey { - case BN = "BatchNormalization" + case norm } /// @@ -150,7 +150,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation activation: activation, params: params) - _bn = BatchNormalizationBase(self) + _norm = LayerWeightsStatsNormalization(self) } /// @@ -178,7 +178,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation params: params) if bn { - _bn = BatchNormalizationBase(self) + _norm = LayerWeightsStatsNormalization(self) } } @@ -193,8 +193,9 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation public required init(from decoder: Decoder) throws { let values = try decoder.container(keyedBy: Keys.self) - _bn = try values.decodeIfPresent(BatchNormalizationBase.self, - forKey: .BN) + _norm = try values.decodeIfPresent( + LayerWeightsStatsNormalization.self, forKey: .norm + ) try super.init(from: decoder) } @@ -212,13 +213,13 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation public override func encode(to encoder: Encoder) throws { var container = encoder.container(keyedBy: Keys.self) - if let bn = _bnGPU + if let norm = _normGPU { - try container.encode(bn, forKey: Keys.BN) + try container.encode(norm, forKey: Keys.norm) } - else if let bn = _bn + else if let norm = _norm { - try container.encode(bn, forKey: Keys.BN) + try container.encode(norm, forKey: Keys.norm) } try super.encode(to: encoder) } @@ -252,19 +253,19 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation ) if inPlace { - layer._bn = _bn - layer._bnGPU = _bnGPU + layer._norm = _norm + layer._normGPU = _normGPU } else { // only one of them should be cloned - if let bn = _bnGPU + if let norm = _normGPU { - layer._bn = bn.clone() + layer._norm = norm.clone() } - else if let bn = _bn + else if let norm = _norm { - layer._bn = bn.clone() + layer._norm = norm.clone() } } return layer @@ -295,19 +296,19 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation ) if inPlace { - layer._bn = _bn - layer._bnGPU = _bnGPU + layer._norm = _norm + layer._normGPU = _normGPU } else { // only one of them should be cloned - if let bn = _bnGPU + if let norm = _normGPU { - layer._bn = bn.clone() + layer._norm = norm.clone() } - else if let bn = _bn + else if let norm = _norm { - layer._bn = bn.clone() + layer._norm = norm.clone() } } @@ -330,13 +331,13 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation params: params ) // only one of them should be cloned - if let bn = _bnGPU + if let norm = _normGPU { - layer._bn = bn.clone() + layer._norm = norm.clone() } - else if let bn = _bn + else if let norm = _norm { - layer._bn = bn.clone() + layer._norm = norm.clone() } return layer } @@ -349,7 +350,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation public override func resetKernelCPU() { super.resetKernelCPU() - bn?.resetKernel() + norm?.resetKernel() } /// /// Clean state resources in the GPU execution context. @@ -359,7 +360,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation public override func resetKernelGPU() { super.resetKernelGPU() - _bnGPU?.resetKernel() + _normGPU?.resetKernel() } /// @@ -371,19 +372,19 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation { super.initKernelCPU() - if let bn = _bnGPU + if let norm = _normGPU { - _bn = BatchNormalization(bn: bn) + _norm = BatchNormalization(norm: norm) } - else if let bn = _bn + else if let norm = _norm { - _bn = BatchNormalization(bn: bn) + _norm = BatchNormalization(norm: norm) } - bn?.initKernel() + norm?.initKernel() if !GrAI.Loop.gradientChecking { - _bnGPU = nil + _normGPU = nil } } @@ -396,19 +397,19 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation { super.initKernelGPU() - if let bn = _bnGPU + if let norm = _normGPU { - _bnGPU = BatchNormalizationGPU(bn: bn) + _normGPU = BatchNormalizationGPU(norm: norm) } - else if let bn = _bn + else if let norm = _norm { - _bnGPU = BatchNormalizationGPU(bn: bn) + _normGPU = BatchNormalizationGPU(norm: norm) } - _bnGPU?.initKernel(deviceID: deviceID) + _normGPU?.initKernel(deviceID: deviceID) if !GrAI.Loop.gradientChecking { - _bn = nil + _norm = nil } } @@ -419,7 +420,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation /// public func initWeightsCPU() { - bn?.initWeights() + norm?.initWeights() } /// /// Initialize weights in the GPU execution context. @@ -428,7 +429,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation /// public func initWeightsGPU() { - _bnGPU?.initWeights() + _normGPU?.initWeights() } /// @@ -439,7 +440,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation public override func forwardGCCPU() throws { try _forwardGCCPU() - bn!.forwardGC(self) + norm!.forwardGC(self) _activation?.forwardGC(self) } @@ -474,7 +475,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation }}} }} - // Prepare GC for BN weights: Ɣ and β. + // Prepare GC for norm weights: Ɣ and β. for batch in 0.. [IWeightArrays] { var weights = [IWeightArrays]() - if let bn = self.bn + if let norm = self.norm { - weights += bn.collectWeights() + weights += norm.collectWeights() } return weights } @@ -701,7 +702,7 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation /// Get the weights in the GPU execution context. public func collectWeightsGPU() -> [IWeightBuffers] { - return _bnGPU!.collectWeights() + return _normGPU!.collectWeights() } /// @@ -714,9 +715,8 @@ public class BN2D: Activation2D, LayerUpdate, LayerWithActivation /// func getOutsGC(depth: Int, elem: Int) -> [Double] { - var sorties = [Double](repeating: 0.0, - count: batchSize * height * width) - + var outs = [Double](repeating: 0.0, + count: batchSize * height * width) for batch in 0..( + _ data: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + if data.count != batchSize * nbChannels * height * width + { + throw LayerError.DataSize + } + if nbChannels != self.nbChannels || + height != self.height || + width != self.width + { + throw LayerError.DataSize + } + try checkStateCPU(batchSize: batchSize) + + switch format + { + case .RGB: + for elem in 0..( + _ data: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + if data.count != batchSize * nbChannels * height * width + { + throw LayerError.DataSize + } + if nbChannels != self.nbChannels || + height != self.height || + width != self.width + { + throw LayerError.DataSize + } + try checkStateForwardGPU(batchSize: batchSize) + + // Wait for previous loop to end to avoid race condition with + // didModifyRange in the following example: + // Convolution.backwardWeightsGPU accesses layerPrev.outs. + MetalKernel.get.download([outs]) + + let outsPtr = outs.shared.buffer + switch format + { + case .RGB: + for elem in 0.., + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + if data.nbElems > batchSize * nbChannels * height * width + { + throw LayerError.DataSize + } + if nbChannels != self.nbChannels || + height != self.height || + width != self.width + { + throw LayerError.DataSize + } + try checkStateForwardGPU(batchSize: batchSize) + outs = data + } } diff --git a/Sources/GrAIdient/Layer2D/Base/LayerOutput2D.swift b/Sources/GrAIdient/Layer2D/Base/LayerOutput2D.swift new file mode 100644 index 00000000..3e1cf343 --- /dev/null +++ b/Sources/GrAIdient/Layer2D/Base/LayerOutput2D.swift @@ -0,0 +1,429 @@ +// +// LayerOutput2D.swift +// GrAIdient +// +// Created by Jean-François Reboud on 04/03/2023. +// + +/// Loss layer of a model with a 2D shape neural structure. +open class LayerOutput2D: Layer2D +{ + /// Coefficient to be applied to the loss computation. + public var coeff: Double = 1.0 + + /// + /// Ground truth buffer in the GPU execution context. + /// Shape ~ (batch, nbChannels, height, width). + /// + public internal(set) var groundTruth: MetalSharedBuffer! = nil + + /// + /// Loss buffer in the GPU execution context. + /// Shape ~ (batch,). + /// + public internal(set) var loss: MetalSharedBuffer! = nil + + private enum Keys: String, CodingKey + { + case coeff + } + + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer2D, params: GrAI.Model.Params) throws + { + super.init(layerPrev: layerPrev, + nbChannels: layerPrev.nbChannels, + height: layerPrev.height, + width: layerPrev.width, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let container = try decoder.container(keyedBy: Keys.self) + let coeff = try container.decode(Float.self, forKey: .coeff) + self.coeff = Double(coeff) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + try container.encode(Float(coeff), forKey: .coeff) + try super.encode(to: encoder) + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We clean the neurons' state (forward and backward). + /// + open override func resetKernelGPU() + { + super.resetKernelGPU() + groundTruth = nil + loss = nil + } + + /// + /// Check and setup ground truth in the CPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// + public func checkGroundTruthCPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + if groundTruth.count != batchSize * nbChannels * height * width + { + throw LayerError.DataSize + } + if batchSize != self.batchSize || + nbChannels != self.nbChannels || + height != self.height || + width != self.width + { + throw LayerError.DataSize + } + if batchSize <= 0 || batchSize > neurons.first!.get(0, 0)!.v.count + { + throw LayerError.BatchSize + } + } + + /// + /// Check and setup ground truth in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: The data format. + /// + public func checkGroundTruthGPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + if groundTruth.count != batchSize * nbChannels * height * width + { + throw LayerError.DataSize + } + if batchSize != self.batchSize || + nbChannels != self.nbChannels || + height != self.height || + width != self.width + { + throw LayerError.DataSize + } + + if self.groundTruth == nil + { + self.groundTruth = MetalSharedBuffer( + batchSize * nbChannels * height * width, + deviceID: deviceID + ) + } + else if batchSize <= 0 || + batchSize * nbChannels * height * width > self.groundTruth.nbElems + { + throw LayerError.BatchSize + } + + let bufferPtr = self.groundTruth.buffer + switch format + { + case .RGB: + for elem in 0.., + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + if batchSize <= 0 || + batchSize * nbChannels * height * width > groundTruth.nbElems + { + throw LayerError.DataSize + } + if batchSize != self.batchSize || + nbChannels != self.nbChannels || + height != self.height || + width != self.width + { + throw LayerError.DataSize + } + } + + /// + /// Setup loss state in the GPU execution context. + /// + /// Throw an error if batch size or ground truth are incoherent. + /// + /// - Parameter batchSize: The batch size of data. + /// + public func checkLossGPU(batchSize: Int) throws + { + if loss == nil + { + loss = MetalSharedBuffer(batchSize, deviceID: deviceID) + } + else if batchSize <= 0 || batchSize > loss.nbElems + { + throw LayerError.BatchSize + } + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + open override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for depth in 0..! = nil + + /// Whether to compute weights' gradients or not. + public var computeDeltaWeights: Bool = true + + /// Whether gradients of weights must be accumulated or not. + public var accumulateDeltaWeights: Bool = false + + /// Cache for weights before calling `initKernel` API. + var _weightsList = [Float]() + + /// Weights in the CPU execution context. + public var weightsCPU: [Float] + { + get { + if _wArrays == nil + { + return _weightsList + } + + var weightsTmp = [Float]() + for depth in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = Constant2D( + nbChannels: nbChannels, + height: height, + width: width, + params: params + ) + if inPlace + { + layer._wArrays = _wArrays + layer._wBuffers = _wBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Resize this layer. + /// + /// - Parameters: + /// - imageWidth: New size width. + /// - imageHeight: New size height. + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// + /// - Returns: A new instance of `Layer`. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public func resize( + imageWidth: Int, + imageHeight: Int, + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = Constant2D( + nbChannels: nbChannels, + height: imageHeight, + width: imageWidth, + params: params + ) + if inPlace + { + layer._wArrays = _wArrays + layer._wBuffers = _wBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + _wArrays?.reset() + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + + _wDeltaWeights = nil + _wBuffers?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + public func initWeightsCPU() + { + _wArrays = WeightArrays(nbChannels) + + if _weightsList.count == 0 + { + for depth in 0..( + batchSize * nbChannels, deviceID: deviceID + ) + } + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: batchSize) + + let newGC = 2 * nbLearnedGC + for depth in 0.. [IWeightArrays] + { + return [_wArrays] + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + return [_wBuffers] + } + + /// + /// Get the weights' gradients in the GPU execution context. + /// + /// Throw an error when layer has not been updated through backward pass or + /// when gradients per sample have not been computed. + /// + /// - Parameter elem: The batch element to retrieve the outputs from. + /// + public func getDeltaWeightsGPU(elem: Int) throws + -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + if !GrAI.Gradient.sample + { + throw UpdateError.PerSample + } + + var deltaWeights = [T]() + MetalKernel.get.download([_wDeltaWeights]) + let deltaWeightsPtr = _wDeltaWeights.shared.buffer + + for depth in 0..() throws -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + + var deltaWeights = [T]() + for depth in 0..() throws -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + + var deltaWeights = [T]() + MetalKernel.get.download([_wBuffers.g_p!]) + let deltaWeightsPtr = _wBuffers.g_p!.shared.buffer + + for i in 0..<_wBuffers.nbElems + { + deltaWeights.append(T(deltaWeightsPtr[i])) + } + return deltaWeights + } +} diff --git a/Sources/GrAIdient/Layer2D/Convolution2D.swift b/Sources/GrAIdient/Layer2D/Convolution2D.swift index 2dfe5166..548b0d4f 100644 --- a/Sources/GrAIdient/Layer2D/Convolution2D.swift +++ b/Sources/GrAIdient/Layer2D/Convolution2D.swift @@ -16,8 +16,14 @@ import MetalKit /// /// The implementation here corresponds to the half padding version of the link below: /// https://github.com/vdumoulin/conv_arithmetic/blob/master/README.md +/// In the PyTorch documentation, we have padding = floor(kernel / 2) and dilation = 1: +/// https://pytorch.org/docs/stable/generated/torch.nn.Conv2d.html /// -public class Convolution2D: BN2D +/// The most standard way is to use an odd kernel size. +/// With a stride of 1, this will preserve the previous layer's size. +/// With a greater stride, this will divide the previous layer's size by stride. +/// +public class Convolution2D: BN2D, LayerWeightInit { /// Downscale factor of the resolution (height and width). let _stride: Int @@ -260,18 +266,25 @@ public class Convolution2D: BN2D } } - /// Get the coefficient to apply during the weights initialization. - var coeffInitWeights: Double + /// Method used to initialize weights values. + public var weightInitClass: WeightInitClass = .XavierUniform + + /// Number of weights values (not considering the biases). + public var weightListSize: Int { get { - if let activation = _activation - { - return activation.coeffInitWeights( - nPrev: nbChannelsPrev * weightHeight * weightWidth, - nCur: nbChannels) - } - return sqrt(2.0 / - Double(nbChannelsPrev * weightHeight * weightWidth)) + return nbChannels * nbChannelsPrev * weightHeight * weightWidth + } + } + + /// Get the number of input and output connections. + public var connectivityIO: (Int, Int) + { + get { + return ( + nbChannelsPrev * weightHeight * weightWidth, + nbChannels * weightHeight * weightWidth + ) } } @@ -285,7 +298,7 @@ public class Convolution2D: BN2D { nbGC += nbChannels } - if _bn != nil || _bnGPU != nil + if _norm != nil || _normGPU != nil { nbGC += 2 * nbChannels } @@ -328,19 +341,40 @@ public class Convolution2D: BN2D } } - var _kernelIndices: (Int, Int, Int, Int) + /// + /// Get indices needed to compute kernel convolution patchs. + /// + /// - Returns: + /// - startI: Start index row offset from a pixel target position. + /// - endI: End index row offset from a pixel target position. + /// - startJ: Start index column offset from a pixel target position. + /// - endJ: End index column offset from a pixel target position. + /// - offI: Padding row offset. + /// - offJ: Padding column offset. + /// + /// For a convolution, there are two situations: + /// - odd kernel: Patchs exclusively target the pixels of the previous layer's grid. + /// - even kernel: Patchs first targets are a void border on the left of the previous layer's grid. + /// + /// For a deconvolution: + /// - Patchs first targets are a void border on the left of the previous layer's grid. + /// + var kernelIndices: (Int, Int, Int, Int, Int, Int) { get { let weightHeightHalf = weightHeight / 2 let weightWidthHalf = weightWidth / 2 - let startI = weightWidth % 2 == 1 ? -weightHeightHalf : - -weightHeightHalf+1 + let startI = weightHeight % 2 == 1 ? -weightHeightHalf : + -weightHeightHalf+1 let endI = weightHeightHalf - let startJ = weightHeight % 2 == 1 ? -weightWidthHalf : - -weightWidthHalf+1 + let startJ = weightWidth % 2 == 1 ? -weightWidthHalf : + -weightWidthHalf+1 let endJ = weightWidthHalf - return (startI, endI, startJ, endJ) + let offI = endI + startI + let offJ = endJ + startJ + + return (startI, endI, startJ, endJ, offI, offJ) } } @@ -375,10 +409,14 @@ public class Convolution2D: BN2D let width = layerPrev.width let height = layerPrev.height - let widthRes = width % _stride - let heightRes = height % _stride - let widthNew = widthRes == 0 ? width / _stride : width / _stride + 1 - let heightNew = heightRes == 0 ? height / _stride : height / _stride + 1 + let padding = Int(floor(Double(size) / 2.0)) + + var tmp = Double(width + 2 * padding - size) + tmp = tmp / Double(stride) + 1.0 + let widthNew = Int(floor(tmp)) + tmp = Double(height + 2 * padding - size) + tmp = tmp / Double(stride) + 1.0 + let heightNew = Int(floor(tmp)) nbWeights = nbChannels * layerPrev.nbChannels weightWidth = size @@ -517,7 +555,7 @@ public class Convolution2D: BN2D stride: _stride, activation: _activation?.name, biases: _updateBiases, - bn: _bn != nil || _bnGPU != nil, + bn: _norm != nil || _normGPU != nil, params: params ) if inPlace @@ -526,19 +564,19 @@ public class Convolution2D: BN2D layer._bArrays = _bArrays layer._wBuffers = _wBuffers layer._bBuffers = _bBuffers - layer._bn = _bn - layer._bnGPU = _bnGPU + layer._norm = _norm + layer._normGPU = _normGPU } else { // only one of them should be cloned - if let bn = _bnGPU + if let bn = _normGPU { - layer._bn = bn.clone() + layer._norm = bn.clone() } - else if let bn = _bn + else if let bn = _norm { - layer._bn = bn.clone() + layer._norm = bn.clone() } if GrAI.Opti.GPU @@ -587,8 +625,8 @@ public class Convolution2D: BN2D layer._bArrays = _bArrays layer._wBuffers = _wBuffers layer._bBuffers = _bBuffers - layer._bn = nil - layer._bnGPU = nil + layer._norm = nil + layer._normGPU = nil } else { @@ -678,6 +716,12 @@ public class Convolution2D: BN2D /// public override func initWeightsCPU() { + if _weightsList.count == 0 + { + _weightsList = generateWeightsList() + _weightsList += [Float](repeating: 0.0, count: nbChannels) + } + super.initWeightsCPU() _wArrays = [WeightGrids]() @@ -688,56 +732,36 @@ public class Convolution2D: BN2D } _bArrays = WeightArrays(nbChannels) - if _weightsList.count == 0 + for elem in 0..= 0, I1 < heightPrev, J1 >= 0, J1 < widthPrev { var w = Double(weightsPtr[offsetWeights]) @@ -1199,8 +1218,8 @@ public class Convolution2D: BN2D let offsetWeights = l-startJ + (offsetStartWeights + k-startI) * weightWidth - let I1 = _stride * i + k - let J1 = _stride * j + l + let I1 = _stride * i + k - offI + let J1 = _stride * j + l - offJ if I1 >= 0, I1 < heightPrev, J1 >= 0, J1 < widthPrev { let w = Double(weightsPtr[offsetWeights]) @@ -1222,7 +1241,7 @@ public class Convolution2D: BN2D }}}}} // Prepare GC for BN weights: Ɣ and β. - if _bnGPU != nil { + if _normGPU != nil { for batch in 0..= 0, I1 < heightPrev, J1 >= 0, J1 < widthPrev { let w = Double(weightsPtr[offsetWeights]) @@ -1272,7 +1291,7 @@ public class Convolution2D: BN2D public override func forwardCPU() throws { try _forwardCPU() - bn?.forward(self) + norm?.forward(self) _activation?.forwardCPU(self) } @@ -1283,7 +1302,7 @@ public class Convolution2D: BN2D try checkStateCPU(batchSize: batchSize) let neuronsPrev = layerPrev.neurons - let (startI, endI, startJ, endJ) = _kernelIndices + let (startI, endI, startJ, endJ, offI, offJ) = kernelIndices for elem in 0..( - batchSize * nbChannels * width * height, deviceID: deviceID) - } - let command = MetalKernel.get.createCommand( forwardKernel, deviceID: deviceID ) @@ -1380,7 +1395,7 @@ public class Convolution2D: BN2D public override func backwardCPU() { _activation?.backwardCPU(self) - bn?.backward(self) + norm?.backward(self) _backwardCPU() _backwardWeightsCPU() @@ -1391,7 +1406,7 @@ public class Convolution2D: BN2D if let layerPrev = self.layerPrev as? Layer2D, mustComputeBackward { let neuronsPrev = layerPrev.neurons - let (startI, endI, startJ, endJ) = _kernelIndices + let (startI, endI, startJ, endJ, offI, offJ) = kernelIndices for elem in 0..= 0, I1 < heightPrev, J1 >= 0, J1 < widthPrev { @@ -637,11 +642,11 @@ public class Deconvolution2D: Convolution2D for k in startI...endI { for l in startJ...endJ { - if (i+k-endI) % _stride == 0 && - (j+l-endJ) % _stride == 0 + if (i-k+startI) % _stride == 0 && + (j-l+startJ) % _stride == 0 { - let I1 = (i+k-endI) / _stride - let J1 = (j+l-endJ) / _stride + let I1 = (i-k+startI) / _stride + let J1 = (j-l+startJ) / _stride if I1 >= 0, I1 < heightPrev, J1 >= 0, J1 < widthPrev { @@ -669,7 +674,7 @@ public class Deconvolution2D: Convolution2D }}}}} // Prepare GC for BN weights: Ɣ and β. - if _bn != nil { + if _norm != nil { for batch in 0..= 0, I1 < heightPrev, J1 >= 0, J1 < widthPrev { @@ -724,7 +729,7 @@ public class Deconvolution2D: Convolution2D try checkStateCPU(batchSize: batchSize) let neuronsPrev = layerPrev.neurons - let (startI, endI, startJ, endJ) = _kernelIndices + let (startI, endI, startJ, endJ, _, _) = kernelIndices for elem in 0.., IWeightBuffers } } -/// First layer with a 2D shape neural structure. +/// Input layer with a 2D shape neural structure. public class Input2D: LayerInput2D, LayerResize, LayerUpdate { /// Grid of "weights". @@ -326,24 +326,6 @@ public class Input2D: LayerInput2D, LayerResize, LayerUpdate /// Initialize weights in the GPU execution context. public func initWeightsGPU() {} - /// - /// API to set data in the CPU execution context. - /// - /// Throw an error if data size is not coherent. - /// - /// - Parameters: - /// - data: The data to set. - /// - format: The data format. - /// - public func setDataCPU( - _ data: [[T]], - format: ImageFormat) throws - { - let batchSize = data.count - let dataTmp = data.reduce([], +) - try setDataCPU(dataTmp, batchSize: batchSize, format: format) - } - /// /// API to set data in the CPU execution context. /// @@ -352,66 +334,25 @@ public class Input2D: LayerInput2D, LayerResize, LayerUpdate /// - Parameters: /// - data: The data to set. /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. /// - format: The data format. /// public func setDataCPU( _ data: [T], batchSize: Int, + nbChannels: Int, height: Int, width: Int, format: ImageFormat) throws { - try checkStateCPU(batchSize: batchSize) - - switch format - { - case .RGB: - for elem in 0..( - _ data: [[T]], - format: ImageFormat) throws - { - let batchSize = data.count - let dataTmp = data.reduce([], +) - try setDataGPU(dataTmp, batchSize: batchSize, format: format) + try checkInputCPU( + data, + batchSize: batchSize, + nbChannels: nbChannels, + height: height, + width: width, + format: format + ) } /// @@ -422,58 +363,25 @@ public class Input2D: LayerInput2D, LayerResize, LayerUpdate /// - Parameters: /// - data: The data to set. /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. /// - format: The data format. /// public func setDataGPU( _ data: [T], batchSize: Int, + nbChannels: Int, height: Int, width: Int, format: ImageFormat) throws { - try checkStateForwardGPU(batchSize: batchSize) - - // Wait for previous loop to end to avoid race condition with - // didModifyRange in the following example: - // Convolution.backwardWeightsGPU accesses layerPrev.outs. - MetalKernel.get.download([outs]) - - let outsPtr = outs.shared.buffer - switch format - { - case .RGB: - for elem in 0.., - batchSize: Int) throws + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws { - try checkStateForwardGPU(batchSize: batchSize) - outs = data + try checkInputGPU( + data, + batchSize: batchSize, + nbChannels: nbChannels, + height: height, + width: width + ) } /// diff --git a/Sources/GrAIdient/Layer2D/InstanceNorm2D.swift b/Sources/GrAIdient/Layer2D/InstanceNorm2D.swift new file mode 100644 index 00000000..ce159f7e --- /dev/null +++ b/Sources/GrAIdient/Layer2D/InstanceNorm2D.swift @@ -0,0 +1,747 @@ +// +// InstanceNorm2D.swift +// GrAIdient +// +// Created by Jean-François Reboud on 17/02/2023. +// + +/// Layer with a 2D shape neural structure, an activation function and instance normalization units. +public class InstanceNorm2D: Activation2D, LayerUpdate, LayerWithActivation +{ + /// Instance normalization by default or instance normalization in the CPU execution context. + var _norm: LayerWeightsNormalization? = nil + /// Instance normalization in the GPU execution context. + var _normGPU: InstanceNormalizationGPU? = nil + + /// Whether to compute weights' gradients or not. + public var computeDeltaWeights: Bool = true + + /// Whether gradients of weights must be accumulated or not. + public var accumulateDeltaWeights: Bool = false + + /// Weights in the CPU execution context. + public var weightsCPU: [Float] + { + get { + var weightsTmp = [Float]() + if let norm = _norm + { + weightsTmp += norm.weights + } + return weightsTmp + } + set { + if let norm = _norm + { + norm.weights = newValue + } + } + } + + /// Weights in the GPU execution context. + public var weightsGPU: [Float] + { + get { + var weightsTmp = [Float]() + if let norm = _normGPU + { + weightsTmp += norm.weights + } + else if let norm = _norm + { + weightsTmp += norm.weights + } + return weightsTmp + } + set { + if let norm = _normGPU + { + norm.weights = newValue + } + else if let norm = _norm + { + norm.weights = newValue + } + } + } + + /// Get instance normalization in the CPU execution context. + var norm: InstanceNormalization? + { + get { + return _norm as? InstanceNormalization + } + } + + /// Number of new weights due to this layer, estimated during the Gradient Checking. + var nbLearnedGC: Int + { + get { + return 2 * nbChannels + } + } + + private enum Keys: String, CodingKey + { + case norm + } + + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - activation: The activation function. + /// - params: Contextual parameters linking to the model. + /// + public override init(layerPrev: Layer2D, activation: String?, + params: GrAI.Model.Params) + { + super.init(layerPrev: layerPrev, + nbChannels: layerPrev.nbChannels, + height: layerPrev.height, + width: layerPrev.width, + activation: activation, + params: params) + + _norm = LayerWeightsNormalization(self) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let values = try decoder.container(keyedBy: Keys.self) + _norm = try values.decodeIfPresent( + LayerWeightsNormalization.self, forKey: .norm + ) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + if let norm = _normGPU + { + try container.encode(norm, forKey: Keys.norm) + } + else if let norm = _norm + { + try container.encode(norm, forKey: Keys.norm) + } + try super.encode(to: encoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = InstanceNorm2D( + layerPrev: layerPrev, + activation: _activation?.name, + params: params + ) + if inPlace + { + layer._norm = _norm + layer._normGPU = _normGPU + } + else + { + // only one of them should be cloned + if let norm = _normGPU + { + layer._norm = norm.clone() + } + else if let norm = _norm + { + layer._norm = norm.clone() + } + } + return layer + } + + /// + /// Extract main operation of this layer without the activation part. + /// + /// This API will create a new layer in the same context as this. + /// + /// - Parameter inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new instance of `Layer`. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public func removeActivation(inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = self.layerPrev as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = InstanceNorm2D( + layerPrev: layerPrev, + activation: nil, + params: params + ) + if inPlace + { + layer._norm = _norm + layer._normGPU = _normGPU + } + else + { + // only one of them should be cloned + if let norm = _normGPU + { + layer._norm = norm.clone() + } + else if let norm = _norm + { + layer._norm = norm.clone() + } + } + + return layer + } + + /// + /// Extract main operation of this layer without the activation part. + /// + /// - Parameter params: Contextual parameters linking to the model. + /// + /// - Returns: A new layer. + /// + public func removeActivation(params: GrAI.Model.Params) -> Layer + { + let layerPrev = self.layerPrev as! Layer2D + let layer = InstanceNorm2D( + layerPrev: layerPrev, + activation: nil, + params: params + ) + // only one of them should be cloned + if let norm = _normGPU + { + layer._norm = norm.clone() + } + else if let norm = _norm + { + layer._norm = norm.clone() + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We reset batch normalization. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + norm?.resetKernel() + } + /// + /// Clean state resources in the GPU execution context. + /// + /// We reset batch normalization. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + _normGPU?.resetKernel() + } + + /// + /// Initialize hard resources in the CPU execution context. + /// + /// We initialize batch normalization. + /// + public override func initKernelCPU() + { + super.initKernelCPU() + + if let norm = _normGPU + { + _norm = InstanceNormalization(norm: norm) + } + else if let norm = _norm + { + _norm = InstanceNormalization(norm: norm) + } + + if !GrAI.Loop.gradientChecking + { + _normGPU = nil + } + } + + /// + /// Initialize hard resources in the GPU execution context. + /// + /// We initialize batch normalization. + /// + public override func initKernelGPU() + { + super.initKernelGPU() + + if let norm = _normGPU + { + _normGPU = InstanceNormalizationGPU(norm: norm) + } + else if let norm = _norm + { + _normGPU = InstanceNormalizationGPU(norm: norm) + } + _normGPU?.initKernel(deviceID: deviceID) + + if !GrAI.Loop.gradientChecking + { + _norm = nil + } + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// We initialize batch normalization's weights. + /// + public func initWeightsCPU() + { + norm?.initWeights() + } + /// + /// Initialize weights in the GPU execution context. + /// + /// We initialize batch normalization's weights. + /// + public func initWeightsGPU() + { + _normGPU?.initWeights() + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try _forwardGCCPU() + norm!.forwardGC(self) + _activation?.forwardGC(self) + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + private func _forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + let newGC = nbGC + 2 * nbLearnedGC + for j in 0.. [IWeightArrays] + { + var weights = [IWeightArrays]() + if let norm = self.norm + { + weights += norm.collectWeights() + } + return weights + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + return _normGPU!.collectWeights() + } + + /// + /// Get the outputs of Gradient Checking (result of the forward pass) in the CPU execution context. + /// + /// - Parameters: + /// - depth: Channel index. + /// - batch: Index of sample in the mini batch. + /// - elem: Weight estimation index during the Gradient Checking. + /// - Returns: The outputs. + /// + func getOutsGC(depth: Int, batch: Int, elem: Int) -> [Double] + { + var outs = [Double](repeating: 0.0, count: height * width) + for i in 0.. [Double] + { + var outs = [Double](repeating: 0.0, count: height * width) + for i in 0.. [Double] + { + var delta = [Double](repeating: 0.0, count: height * width) + for i in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = try! MSE2D(layerPrev: layerPrev, params: params) + layer.coeff = self.coeff + + return layer + } + + /// + /// Estimate the gradients of weights thanks to Gradient Checking. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: The data format. + /// - Returns: The estimated gradients of weights. + /// + public func collectGradientsApprox( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> [T] + { + var gradients = [T]() + let nbGradients = neurons.first!.get(0)!.nbGC / 2 + for elem in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + elem: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + var losses = [T](repeating: 0.0, count: batchSize) + switch format + { + case .RGB: + for batch in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + var losses = [T](repeating: 0.0, count: batchSize) + switch format + { + case .RGB: + for elem in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws -> T + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width, + format: format + ) + return try T(getLossGPU( + self.groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + )) + } + + /// + /// Get loss in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - Returns: The loss value. + /// + public func getLossGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws -> Float + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + try checkLossGPU(batchSize: batchSize) + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + + let command = MetalKernel.get.createCommand( + "MSE2DLoss", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBuffer(loss.metal, atIndex: 5) + + command.dispatchThreads(batchSize) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for i in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + try checkGroundTruthCPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + if let layerPrev = self.layerPrev as? Layer2D, mustComputeBackward + { + let neuronsPrev = layerPrev.neurons + switch format + { + case .RGB: + for elem in 0..( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width, + format: format + ) + try lossDerivativeGPU( + self.groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + } + + /// + /// Compute the derivative of the loss in the GPU execution context. + /// + /// This function is necessary to initialize the backward pass ! + /// In a way, it plays a similar role as the `setData` of the first layer. + /// + /// The `setData` API sets data to the first layer to initialize the forward pass. + /// Here we use the `groundTruth` to initialize the backward pass. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// + public func lossDerivativeGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + try checkGroundTruthGPU( + groundTruth, + batchSize: batchSize, + nbChannels: nbChannels, height: height, width: width + ) + + if let layerPrev = self.layerPrev as? Layer2D, mustComputeBackward + { + try layerPrev.checkStateBackwardGPU(batchSize: batchSize) + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pCoeff: [Float] = [Float(coeff)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pDirty: [UInt32] = layerPrev.dirty ? [1] : [0] + + let command = MetalKernel.get.createCommand( + "MSE2DLossDerivative", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBuffer(groundTruth.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBytes(pCoeff, atIndex: 4) + command.setBytes(pNbBatch, atIndex: 5) + command.setBytes(pDirty, atIndex: 6) + command.setBuffer(layerPrev.delta.metal, atIndex: 7) + + command.dispatchThreads( + width: nbChannels * width, + height: batchSize * height + ) + command.enqueue() + + propagateDirty() + } + } +} diff --git a/Sources/GrAIdient/Layer2D/Multiply2D.swift b/Sources/GrAIdient/Layer2D/Multiply2D.swift index e66330cc..d5d879ec 100644 --- a/Sources/GrAIdient/Layer2D/Multiply2D.swift +++ b/Sources/GrAIdient/Layer2D/Multiply2D.swift @@ -26,7 +26,7 @@ public class Multiply2D: LayerMerge2D /// - layersPrev: List of previous layers that have been queued to the model. /// - params: Contextual parameters linking to the model. /// - public init(layersPrev: [Layer2D], params: GrAI.Model.Params) + public init(layersPrev: [Layer2D], params: GrAI.Model.Params) throws { let layer0 = layersPrev[0] for layerPrev in layersPrev @@ -35,7 +35,7 @@ public class Multiply2D: LayerMerge2D layerPrev.height != layer0.height || layerPrev.width != layer0.width { - fatalError("Layer structure error.") + throw LayerError.Init(message: "Layer structure error.") } } @@ -85,7 +85,7 @@ public class Multiply2D: LayerMerge2D layersPrev.append(mapping[idPrev] as! Layer2D) } - let layer = Multiply2D(layersPrev: layersPrev, params: params) + let layer = try! Multiply2D(layersPrev: layersPrev, params: params) return layer } diff --git a/Sources/GrAIdient/Layer2D/Normalize2D.swift b/Sources/GrAIdient/Layer2D/Normalize2D.swift new file mode 100644 index 00000000..6ad35e3d --- /dev/null +++ b/Sources/GrAIdient/Layer2D/Normalize2D.swift @@ -0,0 +1,780 @@ +// +// Normalize2D.swift +// GrAIdient +// +// Created by Jean-François Reboud on 14/05/2023. +// + +import Foundation +import MetalKit + +/// +/// Layer with a 2D shape neural structure. +/// +/// This layer normalizes thanks to an aggregation on the channel axis of the previous layer. +/// +public class Normalize12D: Layer2D +{ + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer2D, params: GrAI.Model.Params) + { + super.init(layerPrev: layerPrev, + nbChannels: layerPrev.nbChannels, + height: layerPrev.height, + width: layerPrev.width, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + try super.init(from: decoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = Normalize12D( + layerPrev: layerPrev, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for depth in 0.. 1e-12 + { + var sum = 0.0 + for depth1 in 0..! = nil + /// + /// Temporary delta buffer used in the GPU execution context. + /// Shape ~ (batch, nbThreadgroups). + /// + private var _deltaTmp: MetalPrivateBuffer! = nil + + /// Number of thread groups in the GPU execution context. + var nbThreadgroups: Int + { + get { + let value = Double(nbChannels * height * width) / + Double(_threadsPerThreadgroup) + return Int(ceil(value)) + } + } + + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer2D, params: GrAI.Model.Params) + { + super.init(layerPrev: layerPrev, + nbChannels: layerPrev.nbChannels, + height: layerPrev.height, + width: layerPrev.width, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + try super.init(from: decoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = Normalize122D( + layerPrev: layerPrev, + params: params + ) + return layer + } + + /// + /// Initialize state resources in the GPU execution context. + /// + /// We initialize the neurons' forward state. + /// + public override func checkStateForwardGPU(batchSize: Int) throws + { + if _squaredNorm == nil + { + _squaredNorm = MetalPrivateBuffer( + batchSize * nbThreadgroups, deviceID: deviceID + ) + } + try super.checkStateForwardGPU(batchSize: batchSize) + } + + /// + /// Initialize state resources in the GPU execution context. + /// + /// We initialize the neurons' backward state. + /// + public override func checkStateBackwardGPU(batchSize: Int) throws + { + if _deltaTmp == nil + { + _deltaTmp = MetalPrivateBuffer( + batchSize * nbThreadgroups, deviceID: deviceID + ) + } + try super.checkStateBackwardGPU(batchSize: batchSize) + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for depth in 0.. 1e-12 + { + var sum = 0.0 + for i1 in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = SelfCorrelate2D( + layerPrev: layerPrev, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for i in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = try! SimilarityBatchError2D( + layerPrev: layerPrev, params: params + ) + layer.coeff = self.coeff + + return layer + } + + /// + /// Check and setup ground truth in the CPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// + public override func checkGroundTruthCPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + fatalError("Not implemented.") + } + + /// + /// Setup groundTruth state in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: The data format. + /// + public override func checkGroundTruthGPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + fatalError("Not implemented.") + } + + /// + /// Check and setup ground truth in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// + public override func checkGroundTruthGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + fatalError("Not implemented.") + } + + /// + /// Setup loss state in the GPU execution context. + /// + /// Throw an error if batch size or ground truth are incoherent. + /// + /// - Parameter batchSize: The batch size of data. + /// + public override func checkLossGPU(batchSize: Int) throws + { + if loss == nil + { + loss = MetalSharedBuffer( + batchSize * batchSize, + deviceID: deviceID + ) + } + else if batchSize <= 0 || batchSize * batchSize > loss.nbElems + { + throw LayerError.BatchSize + } + } + + /// + /// Estimate the gradients of weights thanks to Gradient Checking. + /// + /// - Returns: The estimated gradients of weights. + /// + public func collectGradientsApprox() -> [T] + { + var gradients = [T]() + let nbGradients = neurons.first!.get(0)!.nbGC / 2 + for elem in 0..(elem: Int) -> T + { + var loss = T(0.0) + for batch1 in 0..() -> T + { + var loss = T(0.0) + for elem1 in 0.. Float + { + try checkLossGPU(batchSize: batchSize) + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + + let command = MetalKernel.get.createCommand( + "similarBatchError2DLoss", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBytes(pNbChannels, atIndex: 1) + command.setBytes(pDimensions, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBuffer(loss.metal, atIndex: 4) + + command.dispatchThreads(width: batchSize, height: batchSize) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for elem1 in 0..! = nil + + /// Batch size sum in the previous layers. + public var mergedBatchSize: Int + { + get { + var sum = 0 + for layerPrev in _layersPrev + { + sum += layerPrev.batchSize + } + return sum + } + } + + private enum Keys: String, CodingKey + { + case coeff + } + + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layersPrev: List of previous layers that have been queued to the model. + /// - params: Contextual parameters linking to the model. + /// + public init(layersPrev: [Layer2D], params: GrAI.Model.Params) throws + { + let layer0 = layersPrev[0] + for layerPrev in layersPrev + { + if layerPrev.nbChannels != 1 + { + throw LayerError.Init( + message: "Previous layer should have only 1 channel." + ) + } + if layerPrev.height != layer0.height || + layerPrev.width != layer0.width + { + throw LayerError.Init(message: "Layer structure error.") + } + } + + super.init(layersPrev: layersPrev, + nbChannels: 1, + height: layer0.height, + width: layer0.width, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let container = try decoder.container(keyedBy: Keys.self) + let coeff = try container.decode(Float.self, forKey: .coeff) + self.coeff = Double(coeff) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + try container.encode(Float(coeff), forKey: .coeff) + try super.encode(to: encoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + var layersPrev = [Layer2D]() + for idPrev in _idsPrev + { + layersPrev.append(mapping[idPrev] as! Layer2D) + } + + let layer = try! SimilarityError2D( + layersPrev: layersPrev, params: params + ) + layer.coeff = self.coeff + + return layer + } + + /// + /// Setup loss state in the GPU execution context. + /// + /// Throw an error if batch size or ground truth are incoherent. + /// + /// - Parameter batchSize: The batch size of data. + /// + public func checkLossGPU(batchSize: Int) throws + { + if loss == nil + { + loss = MetalSharedBuffer( + batchSize * batchSize, + deviceID: deviceID + ) + } + else if batchSize <= 0 || batchSize * batchSize > loss.nbElems + { + throw LayerError.BatchSize + } + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + open override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: mergedBatchSize) + + let (nbSameElems, layersIndex, nbElems) = getMergedGraph() + + var nbGC = nbSameElems + for nbElemsTmp in nbElems + { + nbGC += nbElemsTmp + } + + for i in 0..() -> [T] + { + var gradients = [T]() + let nbGradients = neurons.first!.get(0)!.nbGC / 2 + for elem in 0..(elem: Int) -> T + { + var loss = T(0.0) + for batch1 in 0..() -> T + { + var loss = T(0.0) + for elem1 in 0.. Float + { + try checkLossGPU(batchSize: mergedBatchSize) + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pNbBatch: [UInt32] = [UInt32(mergedBatchSize)] + + let command = MetalKernel.get.createCommand( + "similarBatchError2DLoss", deviceID: deviceID + ) + command.setBuffer(outs.metal, atIndex: 0) + command.setBytes(pNbChannels, atIndex: 1) + command.setBytes(pDimensions, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBuffer(loss.metal, atIndex: 4) + + command.dispatchThreads( + width: mergedBatchSize, + height: mergedBatchSize + ) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for elem1 in 0..: Codable +{ + let min: T + let max: T + + /// + /// Create the bounded interval. + /// + /// Throw an error when parameter values are not coherent. + /// + /// - Parameters: + /// - min: The minimum value of the interval. + /// - max: The maximum value of the interval. + /// + public init(min: T, max: T) throws + { + self.min = min + self.max = max + if max < min + { + throw RangeError.ValueError + } + } +} + +/// +/// Layer with a 2D shape neural structure. +/// +/// This layer add some noise (in the HSV space) to the RGB channels of the previous layer. +/// +public class ColorJitterHSV: Layer2D +{ + let _rangeH: Range + let _rangeS: Range + let _rangeV: Range + + private enum Keys: String, CodingKey + { + case rangeH + case rangeS + case rangeV + } + + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - rangeH: Range of noise in the hue dimension. + /// - rangeS: Range of noise in the saturation dimension. + /// - rangeV: Range of noise in the value dimension. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer2D, + rangeH: Range, + rangeS: Range, + rangeV: Range, + params: GrAI.Model.Params) throws + { + _rangeH = rangeH + _rangeS = rangeS + _rangeV = rangeV + + let width = layerPrev.width + let height = layerPrev.height + let nbChannels = layerPrev.nbChannels + + if nbChannels != 3 + { + throw LayerError.Init( + message: "Previous layer should have 3 channels: RGB." + ) + } + + super.init(layerPrev: layerPrev, + nbChannels: nbChannels, + height: height, + width: width, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let values = try decoder.container(keyedBy: Keys.self) + _rangeH = try values.decode(Range.self, forKey: Keys.rangeH) + _rangeS = try values.decode(Range.self, forKey: Keys.rangeS) + _rangeV = try values.decode(Range.self, forKey: Keys.rangeV) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + try container.encode(_rangeH, forKey: Keys.rangeH) + try container.encode(_rangeS, forKey: Keys.rangeS) + try container.encode(_rangeV, forKey: Keys.rangeV) + try super.encode(to: encoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = try! ColorJitterHSV( + layerPrev: layerPrev, + rangeH: _rangeH, + rangeS: _rangeS, + rangeV: _rangeV, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + fatalError("Not implemented.") + } + + /// + /// Apply the forward pass of the Gradient Checking in GPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCGPU() throws + { + try forwardGCCPU() + } + + /// + /// Apply the forward pass in the CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let noiseH = Double.random(in: _rangeH.min..._rangeH.max) + let noiseS = Double.random(in: _rangeS.min..._rangeS.max) + let noiseV = Double.random(in: _rangeV.min..._rangeV.max) + + let neuronsPrev = layerPrev.neurons + for elem in 0..= cropDimension || offsetJ >= cropDimension { - fatalError( + throw LayerError.Init(message: """ `offsetI` and `offsetJ` should be lower than `cropDimension` and higher than 0. @@ -177,7 +177,7 @@ public class Crop2D: Layer2D let layer: Crop2D if !_doNotRandom { - layer = Crop2D( + layer = try! Crop2D( layerPrev: layerPrev, cropDimension: _cropDimension, params: params @@ -185,7 +185,7 @@ public class Crop2D: Layer2D } else { - layer = Crop2D( + layer = try! Crop2D( layerPrev: layerPrev, cropDimension: _cropDimension, offsetI: _offsetI, diff --git a/Sources/GrAIdient/Layer2D/DecorrelateRGB.swift b/Sources/GrAIdient/Layer2D/Transform/DecorrelateRGB.swift similarity index 97% rename from Sources/GrAIdient/Layer2D/DecorrelateRGB.swift rename to Sources/GrAIdient/Layer2D/Transform/DecorrelateRGB.swift index 916c2321..c72a02d6 100644 --- a/Sources/GrAIdient/Layer2D/DecorrelateRGB.swift +++ b/Sources/GrAIdient/Layer2D/Transform/DecorrelateRGB.swift @@ -29,7 +29,7 @@ public class DecorrelateRGB: Layer2D /// public init(layerPrev: Layer2D, correlation: [Double], - params: GrAI.Model.Params) + params: GrAI.Model.Params) throws { _correlation = correlation @@ -39,7 +39,9 @@ public class DecorrelateRGB: Layer2D if nbChannels != 3 { - fatalError("DecorrelateRGB can only be used with 3 channels.") + throw LayerError.Init( + message: "DecorrelateRGB can only be used with 3 channels." + ) } super.init(layerPrev: layerPrev, nbChannels: nbChannels, @@ -105,7 +107,7 @@ public class DecorrelateRGB: Layer2D let params = GrAI.Model.Params(context: context) params.context.curID = id - let layer = DecorrelateRGB( + let layer = try! DecorrelateRGB( layerPrev: layerPrev, correlation: _correlation, params: params diff --git a/Sources/GrAIdient/Layer2D/FTFrequences2D.swift b/Sources/GrAIdient/Layer2D/Transform/FTFrequences2D.swift similarity index 96% rename from Sources/GrAIdient/Layer2D/FTFrequences2D.swift rename to Sources/GrAIdient/Layer2D/Transform/FTFrequences2D.swift index c32927e8..36291527 100644 --- a/Sources/GrAIdient/Layer2D/FTFrequences2D.swift +++ b/Sources/GrAIdient/Layer2D/Transform/FTFrequences2D.swift @@ -23,12 +23,14 @@ public class FTFrequences2D: LayerInput2D, LayerResize /// - params: Contextual parameters linking to the model. /// public init(nbChannels: Int, dimension: Int, - params: GrAI.Model.Params) + params: GrAI.Model.Params) throws { if nbChannels % 2 != 0 { - fatalError("FTFrequences2D input channels " + - "should be a multiple of 2.") + throw LayerError.Init( + message: "FTFrequences2D input channels " + + "should be a multiple of 2." + ) } super.init(layerPrev: nil, nbChannels: nbChannels, @@ -72,7 +74,7 @@ public class FTFrequences2D: LayerInput2D, LayerResize let params = GrAI.Model.Params(context: context) params.context.curID = id - let layer = FTFrequences2D( + let layer = try! FTFrequences2D( nbChannels: nbChannels, dimension: width, params: params ) @@ -107,7 +109,7 @@ public class FTFrequences2D: LayerInput2D, LayerResize let params = GrAI.Model.Params(context: context) params.context.curID = id - let layer = FTFrequences2D( + let layer = try! FTFrequences2D( nbChannels: nbChannels, dimension: imageWidth, params: params ) diff --git a/Sources/GrAIdient/Layer2D/Transform/Flip2D.swift b/Sources/GrAIdient/Layer2D/Transform/Flip2D.swift new file mode 100644 index 00000000..c553b2a1 --- /dev/null +++ b/Sources/GrAIdient/Layer2D/Transform/Flip2D.swift @@ -0,0 +1,503 @@ +// +// Flip2D.swift +// GrAIdient +// +// Created by Jean-François Reboud on 18/05/2023. +// + +/// +/// Layer with a 2D shape neural structure. +/// +/// This layer flips the input horizontally. +/// +public class FlipHorizontal2D: Layer2D +{ + let _probability: Double + + var _doFlip = false + + var forwardKernel: String + { + get { + return "flipHorizontal2DForward" + } + } + var backwardKernel: String + { + get { + return "flipHorizontal2DBackward" + } + } + + private enum Keys: String, CodingKey + { + case probability + } + + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - probability: Probability to flip horizontally. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer2D, + probability: Double, + params: GrAI.Model.Params) + { + _probability = probability + + let width = layerPrev.width + let height = layerPrev.height + let nbChannels = layerPrev.nbChannels + + super.init(layerPrev: layerPrev, + nbChannels: nbChannels, + height: height, + width: width, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let values = try decoder.container(keyedBy: Keys.self) + _probability = try values.decode(Double.self, forKey: Keys.probability) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + try container.encode(_probability, forKey: Keys.probability) + try super.encode(to: encoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = FlipHorizontal2D( + layerPrev: layerPrev, + probability: _probability, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for depth in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = FlipVertical2D( + layerPrev: layerPrev, + probability: _probability, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for depth in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let dimension = Double(layerPrev.height) * _scalesList[0] + let layer = try! ResizeBilinear( + layerPrev: layerPrev, + dimension: Int(round(dimension)), + params: params + ) + return layer + } +} diff --git a/Sources/GrAIdient/Layer2D/Transform/ResizeBilinearCrop.swift b/Sources/GrAIdient/Layer2D/Transform/ResizeBilinearCrop.swift index ab931ab5..40c3faa5 100644 --- a/Sources/GrAIdient/Layer2D/Transform/ResizeBilinearCrop.swift +++ b/Sources/GrAIdient/Layer2D/Transform/ResizeBilinearCrop.swift @@ -17,7 +17,7 @@ import Foundation /// /// When one unique scale is used, there are 2 scenario to consider. /// - if scale >= 1: a crop will be needed to extract patches of the input grids so that -/// the resize of these patches match the expected scale. The final dimensions of the +/// the resize of these patches matches the expected scale. The final dimensions of the /// output grids are exactly the same as the dimensions of the input grids. /// - if scale < 1: no crop is needed. The final dimensions will /// correspond to the scale of the dimensions of the input grids. @@ -25,10 +25,11 @@ import Foundation public class ResizeBilinearCrop: Layer2D { let _scalesList: [Double] + let _minScale: Double? + let _maxScale: Double? var _offsetI: Int = 0 var _offsetJ: Int = 0 - var _doNotRandom: Bool = false var _width2Resize: Int = 0 var _height2Resize: Int = 0 @@ -36,7 +37,8 @@ public class ResizeBilinearCrop: Layer2D private enum Keys: String, CodingKey { case scalesList - case doNotRandom + case minScale + case maxScale case offsetI case offsetJ } @@ -51,19 +53,25 @@ public class ResizeBilinearCrop: Layer2D /// public init(layerPrev: Layer2D, scalesList: [Double], - params: GrAI.Model.Params) + params: GrAI.Model.Params) throws { _scalesList = scalesList + _minScale = nil + _maxScale = nil if scalesList.count == 0 { - fatalError("`scalesList` should have at least one element.") + throw LayerError.Init( + message: "`scalesList` should have at least one element." + ) } for scale in scalesList { if scale == 0 { - fatalError("Only non 0 scales are possible.") + throw LayerError.Init( + message: "Only non 0 scales are possible." + ) } } @@ -81,6 +89,7 @@ public class ResizeBilinearCrop: Layer2D height = min(height, Int(round(scale * Double(heightPrev)))) } } + super.init(layerPrev: layerPrev, nbChannels: nbChannels, height: height, @@ -102,20 +111,23 @@ public class ResizeBilinearCrop: Layer2D scale: Double, offsetI: Int, offsetJ: Int, - params: GrAI.Model.Params) + params: GrAI.Model.Params) throws { _scalesList = [scale] - _doNotRandom = true + _minScale = nil + _maxScale = nil _offsetI = offsetI _offsetJ = offsetJ if scale == 0 { - fatalError("Only non 0 scales are possible.") + throw LayerError.Init(message: "Only non 0 scales are possible.") } if offsetI < 0 || offsetJ < 0 { - fatalError("`offsetI` and `offsetJ` should be higher than 0.") + throw LayerError.Init( + message: "`offsetI` and `offsetJ` should be higher than 0." + ) } let nbChannels = layerPrev.nbChannels @@ -137,6 +149,45 @@ public class ResizeBilinearCrop: Layer2D params: params) } + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - minScale: Minimum scale to apply to (heightPrev, widthPrev) dimensions. + /// - maxScale: Maximum scale to apply to (heightPrev, widthPrev) dimensions. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer2D, + minScale: Double, + maxScale: Double, + params: GrAI.Model.Params) throws + { + _scalesList = [] + _minScale = minScale + _maxScale = maxScale + + if minScale >= maxScale || minScale <= 0.0 + { + throw LayerError.Init(message: "`minScale` is not coherent.") + } + + let nbChannels = layerPrev.nbChannels + let heightPrev = layerPrev.height + let widthPrev = layerPrev.width + + let width = minScale < 1.0 ? + Int(round(minScale * Double(widthPrev))) : widthPrev + let height = minScale < 1.0 ? + Int(round(minScale * Double(heightPrev))) : heightPrev + + super.init(layerPrev: layerPrev, + nbChannels: nbChannels, + height: height, + width: width, + params: params) + } + /// /// Decode from the disk. /// @@ -151,7 +202,12 @@ public class ResizeBilinearCrop: Layer2D _scalesList = try values.decode( [Double].self, forKey: Keys.scalesList ) - _doNotRandom = try values.decode(Bool.self, forKey: Keys.doNotRandom) + _minScale = try values.decodeIfPresent( + Double.self, forKey: Keys.minScale + ) + _maxScale = try values.decodeIfPresent( + Double.self, forKey: Keys.maxScale + ) _offsetI = try values.decode(Int.self, forKey: Keys.offsetI) _offsetJ = try values.decode(Int.self, forKey: Keys.offsetJ) try super.init(from: decoder) @@ -172,7 +228,14 @@ public class ResizeBilinearCrop: Layer2D { var container = encoder.container(keyedBy: Keys.self) try container.encode(_scalesList, forKey: Keys.scalesList) - try container.encode(_doNotRandom, forKey: Keys.doNotRandom) + if let minScale = _minScale + { + try container.encode(minScale, forKey: Keys.minScale) + } + if let maxScale = _maxScale + { + try container.encode(maxScale, forKey: Keys.maxScale) + } try container.encode(_offsetI, forKey: Keys.offsetI) try container.encode(_offsetJ, forKey: Keys.offsetJ) try super.encode(to: encoder) @@ -201,17 +264,17 @@ public class ResizeBilinearCrop: Layer2D params.context.curID = id let layer: ResizeBilinearCrop - if !_doNotRandom + if _scalesList.count > 1 { - layer = ResizeBilinearCrop( + layer = try! ResizeBilinearCrop( layerPrev: layerPrev, scalesList: _scalesList, params: params ) } - else + else if _scalesList.count == 1 { - layer = ResizeBilinearCrop( + layer = try! ResizeBilinearCrop( layerPrev: layerPrev, scale: _scalesList[0], offsetI: _offsetI, @@ -219,6 +282,19 @@ public class ResizeBilinearCrop: Layer2D params: params ) } + else if let minScale = _minScale, let maxScale = _maxScale + { + layer = try! ResizeBilinearCrop( + layerPrev: layerPrev, + minScale: minScale, + maxScale: maxScale, + params: params + ) + } + else + { + fatalError() + } return layer } @@ -319,10 +395,19 @@ public class ResizeBilinearCrop: Layer2D let randIndex = Int.random(in: 0..<_scalesList.count) ratioInOut = _scalesList[randIndex] } - else + else if _scalesList.count == 1 { ratioInOut = _scalesList[0] } + else if let minScale = _minScale, let maxScale = _maxScale + { + ratioInOut = Double.random(in: minScale...maxScale) + } + else + { + fatalError() + } + _width2Resize = Int(floor(Double(width) / ratioInOut)) _height2Resize = Int(floor(Double(height) / ratioInOut)) @@ -334,7 +419,8 @@ public class ResizeBilinearCrop: Layer2D let cropDimensionI = heightPrev - _height2Resize let cropDimensionJ = widthPrev - _width2Resize - if !_doNotRandom + if _scalesList.count > 1 || + (_scalesList.count == 0 && _minScale != nil && _maxScale != nil) { if cropDimensionI == 0 { @@ -353,14 +439,21 @@ public class ResizeBilinearCrop: Layer2D _offsetJ = Int.random(in: 0.. cropDimensionI || _offsetJ > cropDimensionJ + else if _scalesList.count == 1 { - fatalError( - """ - `offsetI` and `offsetJ` should be lower than - `cropDimension`. - """ - ) + if _offsetI > cropDimensionI || _offsetJ > cropDimensionJ + { + fatalError( + """ + `offsetI` and `offsetJ` should be lower than + `cropDimension`. + """ + ) + } + } + else + { + fatalError() } let neuronsPrev = layerPrev.neurons @@ -418,10 +511,19 @@ public class ResizeBilinearCrop: Layer2D let randIndex = Int.random(in: 0..<_scalesList.count) ratioInOut = _scalesList[randIndex] } - else + else if _scalesList.count == 1 { ratioInOut = _scalesList[0] } + else if let minScale = _minScale, let maxScale = _maxScale + { + ratioInOut = Double.random(in: minScale...maxScale) + } + else + { + fatalError() + } + _width2Resize = Int(floor(Double(width) / ratioInOut)) _height2Resize = Int(floor(Double(height) / ratioInOut)) @@ -430,7 +532,8 @@ public class ResizeBilinearCrop: Layer2D let cropDimensionI = heightPrev - _height2Resize let cropDimensionJ = widthPrev - _width2Resize - if !_doNotRandom + if _scalesList.count > 1 || + (_scalesList.count == 0 && _minScale != nil && _maxScale != nil) { if cropDimensionI == 0 { @@ -449,14 +552,21 @@ public class ResizeBilinearCrop: Layer2D _offsetJ = Int.random(in: 0.. cropDimensionI || _offsetJ > cropDimensionJ + else if _scalesList.count == 1 { - fatalError( - """ - `offsetI` and `offsetJ` should be lower than - `cropDimension`. - """ - ) + if _offsetI > cropDimensionI || _offsetJ > cropDimensionJ + { + fatalError( + """ + `offsetI` and `offsetJ` should be lower than + `cropDimension`. + """ + ) + } + } + else + { + fatalError() } let pNbChannels: [UInt32] = [UInt32(nbChannels)] diff --git a/Sources/GrAIdient/Layer2D/Transform/ResizeBilinearPad.swift b/Sources/GrAIdient/Layer2D/Transform/ResizeBilinearPad.swift index b08ca37a..63178539 100644 --- a/Sources/GrAIdient/Layer2D/Transform/ResizeBilinearPad.swift +++ b/Sources/GrAIdient/Layer2D/Transform/ResizeBilinearPad.swift @@ -21,14 +21,47 @@ import Foundation public class ResizeBilinearPad: Layer2D { let _scalesList: [Double] + let _minScale: Double? + let _maxScale: Double? + let _padValue: Double var _widthResize: Int = 0 var _heightResize: Int = 0 + /// + /// Get padding dimensions. + /// + /// - Returns: + /// - startI: Start row padding offset. + /// - endI: End row padding offset. + /// - startJ: Start column padding offset. + /// - endJ: End column padding offset. + /// + var padDimensions: (Int, Int, Int, Int) + { + get { + let blockI = height - _heightResize + let blockJ = width - _widthResize + + let halfBlockI = blockI / 2 + let halfBlockJ = blockJ / 2 + + let startI = blockI % 2 == 0 ? halfBlockI : halfBlockI + 1 + let endI = halfBlockI + + let startJ = blockJ % 2 == 0 ? halfBlockJ : halfBlockJ + 1 + let endJ = halfBlockJ + + return (startI, endI, startJ, endJ) + } + } + private enum Keys: String, CodingKey { case scalesList + case minScale + case maxScale case padValue } @@ -44,20 +77,26 @@ public class ResizeBilinearPad: Layer2D public init(layerPrev: Layer2D, scalesList: [Double], padValue: Double, - params: GrAI.Model.Params) + params: GrAI.Model.Params) throws { _padValue = padValue _scalesList = scalesList + _minScale = nil + _maxScale = nil if scalesList.count == 0 { - fatalError("`scalesList` should have at least one element.") + throw LayerError.Init( + message: "`scalesList` should have at least one element." + ) } for scale in scalesList { if scale == 0 { - fatalError("Only non 0 scales are possible.") + throw LayerError.Init( + message: "Only non 0 scales are possible." + ) } } @@ -72,6 +111,47 @@ public class ResizeBilinearPad: Layer2D width = max(width, Int(round(scale * Double(widthPrev)))) height = max(height, Int(round(scale * Double(heightPrev)))) } + + super.init(layerPrev: layerPrev, + nbChannels: nbChannels, + height: height, + width: width, + params: params) + } + + /// + /// Create a layer with a 2D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - minScale: Minimum scale to apply to (heightPrev, widthPrev) dimensions. + /// - maxScale: Maximum scale to apply to (heightPrev, widthPrev) dimensions. + /// - padValue: Value to set on the created borders. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer2D, + minScale: Double, + maxScale: Double, + padValue: Double, + params: GrAI.Model.Params) throws + { + _padValue = padValue + _scalesList = [] + _minScale = minScale + _maxScale = maxScale + + if minScale >= maxScale || minScale <= 0.0 + { + throw LayerError.Init(message: "`minScale` is not coherent.") + } + + let nbChannels = layerPrev.nbChannels + let heightPrev = layerPrev.height + let widthPrev = layerPrev.width + + let width = Int(round(maxScale * Double(widthPrev))) + let height = Int(round(maxScale * Double(heightPrev))) + super.init(layerPrev: layerPrev, nbChannels: nbChannels, height: height, @@ -93,6 +173,12 @@ public class ResizeBilinearPad: Layer2D _scalesList = try values.decode( [Double].self, forKey: Keys.scalesList ) + _minScale = try values.decodeIfPresent( + Double.self, forKey: Keys.minScale + ) + _maxScale = try values.decodeIfPresent( + Double.self, forKey: Keys.maxScale + ) _padValue = try values.decode( Double.self, forKey: Keys.padValue ) @@ -114,6 +200,14 @@ public class ResizeBilinearPad: Layer2D { var container = encoder.container(keyedBy: Keys.self) try container.encode(_scalesList, forKey: Keys.scalesList) + if let minScale = _minScale + { + try container.encode(minScale, forKey: Keys.minScale) + } + if let maxScale = _maxScale + { + try container.encode(maxScale, forKey: Keys.maxScale) + } try container.encode(_padValue, forKey: Keys.padValue) try super.encode(to: encoder) } @@ -140,12 +234,30 @@ public class ResizeBilinearPad: Layer2D let params = GrAI.Model.Params(context: context) params.context.curID = id - let layer = ResizeBilinearPad( - layerPrev: layerPrev, - scalesList: _scalesList, - padValue: _padValue, - params: params - ) + let layer: ResizeBilinearPad + if _scalesList.count != 0 + { + layer = try! ResizeBilinearPad( + layerPrev: layerPrev, + scalesList: _scalesList, + padValue: _padValue, + params: params + ) + } + else if let minScale = _minScale, let maxScale = _maxScale + { + layer = try! ResizeBilinearPad( + layerPrev: layerPrev, + minScale: minScale, + maxScale: maxScale, + padValue: _padValue, + params: params + ) + } + else + { + fatalError() + } return layer } @@ -177,8 +289,10 @@ public class ResizeBilinearPad: Layer2D let widthPrev = layerPrev.width let ratioInOutI = Double(heightPrev - 1) / Double(_heightResize - 1) let ratioInOutJ = Double(widthPrev - 1) / Double(_widthResize - 1) - let padDimensionI = (height - _heightResize) / 2 - let padDimensionJ = (width - _widthResize) / 2 + let ( + padStartI, padEndI, + padStartJ, padEndJ + ) = padDimensions let neuronsPrev = layerPrev.neurons for batch in 0..= height - padDimensionI || - j < padDimensionJ || j >= width - padDimensionJ + if i < padStartI || i >= height - padEndI || + j < padStartJ || j >= width - padEndJ { neurons[depth].get(i, j)!.gc[batch][elem].out = _padValue } else { - let I = i-padDimensionI - let J = j-padDimensionJ + let I = i-padStartI + let J = j-padStartJ let iPrev = Double(I) * ratioInOutI let jPrev = Double(J) * ratioInOutJ @@ -265,11 +379,25 @@ public class ResizeBilinearPad: Layer2D _widthResize = Int(round(ratioInOut * Double(widthPrev))) _heightResize = Int(round(ratioInOut * Double(heightPrev))) } + else if _scalesList.count == 0, + let minScale = _minScale, let maxScale = _maxScale + { + let ratioInOut = Double.random(in: minScale...maxScale) + + _widthResize = Int(round(ratioInOut * Double(widthPrev))) + _heightResize = Int(round(ratioInOut * Double(heightPrev))) + } + else if _scalesList.count != 1 + { + fatalError() + } let ratioInOutI = Double(heightPrev - 1) / Double(_heightResize - 1) let ratioInOutJ = Double(widthPrev - 1) / Double(_widthResize - 1) - let padDimensionI = (height - _heightResize) / 2 - let padDimensionJ = (width - _widthResize) / 2 + let ( + padStartI, padEndI, + padStartJ, padEndJ + ) = padDimensions let neuronsPrev = layerPrev.neurons for elem in 0..= height - padDimensionI || - j < padDimensionJ || j >= width - padDimensionJ + if i < padStartI || i >= height - padEndI || + j < padStartJ || j >= width - padEndJ { neurons[depth].get(i, j)!.v[elem].out = _padValue } else { - let I = i-padDimensionI - let J = j-padDimensionJ + let I = i-padStartI + let J = j-padStartJ let iPrev = Double(I) * ratioInOutI let jPrev = Double(J) * ratioInOutJ @@ -344,6 +472,23 @@ public class ResizeBilinearPad: Layer2D _widthResize = Int(round(ratioInOut * Double(widthPrev))) _heightResize = Int(round(ratioInOut * Double(heightPrev))) } + else if _scalesList.count == 0, + let minScale = _minScale, let maxScale = _maxScale + { + let ratioInOut = Double.random(in: minScale...maxScale) + + _widthResize = Int(round(ratioInOut * Double(widthPrev))) + _heightResize = Int(round(ratioInOut * Double(heightPrev))) + } + else if _scalesList.count != 1 + { + fatalError() + } + + let ( + padStartI, padEndI, + padStartJ, padEndJ + ) = padDimensions let pNbChannels: [UInt32] = [UInt32(nbChannels)] let pNbBatch: [UInt32] = [UInt32(batchSize)] @@ -354,6 +499,10 @@ public class ResizeBilinearPad: Layer2D let pDimensionsResize: [UInt32] = [ UInt32(_widthResize), UInt32(_heightResize) ] + let pPadDimensions: [UInt32] = [ + UInt32(padStartI), UInt32(padEndI), + UInt32(padStartJ), UInt32(padEndJ) + ] let pPadValue: [Float] = [Float(_padValue)] let command = MetalKernel.get.createCommand( @@ -364,9 +513,10 @@ public class ResizeBilinearPad: Layer2D command.setBytes(pDimensions, atIndex: 2) command.setBytes(pDimensionsPrev, atIndex: 3) command.setBytes(pDimensionsResize, atIndex: 4) - command.setBytes(pPadValue, atIndex: 5) - command.setBytes(pNbBatch, atIndex: 6) - command.setBuffer(outs.metal, atIndex: 7) + command.setBytes(pPadDimensions, atIndex: 5) + command.setBytes(pPadValue, atIndex: 6) + command.setBytes(pNbBatch, atIndex: 7) + command.setBuffer(outs.metal, atIndex: 8) command.dispatchThreads( width: width * nbChannels, @@ -400,8 +550,7 @@ public class ResizeBilinearPad: Layer2D let ratioInOutI = Double(heightPrev - 1) / Double(_heightResize - 1) let ratioInOutJ = Double(widthPrev - 1) / Double(_widthResize - 1) - let padDimensionI = (height - _heightResize) / 2 - let padDimensionJ = (width - _widthResize) / 2 + let (padStartI, _, padStartJ, _) = padDimensions for elem in 0..= maxAngle + { + throw LayerError.Init(message: "`minAngle` is not coherent.") } let nbChannels = layerPrev.nbChannels @@ -71,6 +118,12 @@ public class Rotate2D: Layer2D _anglesList = try values.decode( [Double].self, forKey: Keys.anglesList ) + _minAngle = try values.decodeIfPresent( + Double.self, forKey: Keys.minAngle + ) + _maxAngle = try values.decodeIfPresent( + Double.self, forKey: Keys.maxAngle + ) _padValue = try values.decode( Double.self, forKey: Keys.padValue ) @@ -92,6 +145,14 @@ public class Rotate2D: Layer2D { var container = encoder.container(keyedBy: Keys.self) try container.encode(_anglesList, forKey: Keys.anglesList) + if let minAngle = _minAngle + { + try container.encode(minAngle, forKey: Keys.minAngle) + } + if let maxAngle = _maxAngle + { + try container.encode(maxAngle, forKey: Keys.maxAngle) + } try container.encode(_padValue, forKey: Keys.padValue) try super.encode(to: encoder) } @@ -118,12 +179,30 @@ public class Rotate2D: Layer2D let params = GrAI.Model.Params(context: context) params.context.curID = id - let layer = Rotate2D( - layerPrev: layerPrev, - anglesList: _anglesList, - padValue: _padValue, - params: params - ) + let layer: Rotate2D + if _anglesList.count != 0 + { + layer = try! Rotate2D( + layerPrev: layerPrev, + anglesList: _anglesList, + padValue: _padValue, + params: params + ) + } + else if let minAngle = _minAngle, let maxAngle = _maxAngle + { + layer = try! Rotate2D( + layerPrev: layerPrev, + minAngle: minAngle, + maxAngle: maxAngle, + padValue: _padValue, + params: params + ) + } + else + { + fatalError() + } return layer } @@ -208,8 +287,21 @@ public class Rotate2D: Layer2D { try checkStateCPU(batchSize: batchSize) - let randIndex = Int.random(in: 0..<_anglesList.count) - let angle = _anglesList[randIndex] + let angle: Double + if _anglesList.count != 0 + { + let randIndex = Int.random(in: 0..<_anglesList.count) + angle = _anglesList[randIndex] + } + else if _anglesList.count == 0, + let minAngle = _minAngle, let maxAngle = _maxAngle + { + angle = Double.random(in: minAngle...maxAngle) + } + else + { + fatalError() + } _angle = angle * Double.pi / 180.0 let centerI = Double(height - 1) / 2.0 @@ -257,8 +349,21 @@ public class Rotate2D: Layer2D { try checkStateForwardGPU(batchSize: batchSize) - let randIndex = Int.random(in: 0..<_anglesList.count) - let angle = _anglesList[randIndex] + let angle: Double + if _anglesList.count != 0 + { + let randIndex = Int.random(in: 0..<_anglesList.count) + angle = _anglesList[randIndex] + } + else if _anglesList.count == 0, + let minAngle = _minAngle, let maxAngle = _maxAngle + { + angle = Double.random(in: minAngle...maxAngle) + } + else + { + fatalError() + } _angle = angle * Double.pi / 180.0 let pNbChannels: [UInt32] = [UInt32(nbChannels)] diff --git a/Sources/GrAIdient/Layer2D/VQ2D.swift b/Sources/GrAIdient/Layer2D/VQ2D.swift new file mode 100644 index 00000000..e0fc5ed8 --- /dev/null +++ b/Sources/GrAIdient/Layer2D/VQ2D.swift @@ -0,0 +1,915 @@ +// +// VQ2D.swift +// GrAIdient +// +// Created by Jean-François Reboud on 29/03/2023. +// + +import Foundation + +/// Error occuring during the layer forward or backward propagation. +public enum VQError: Error +{ + /// Could not find a positive index value. + case IndexValue + /// Call to loss API is redundant. + case RedundantLoss +} + +extension VQError: CustomStringConvertible +{ + public var description: String + { + switch self + { + case .IndexValue: + return "Could not find a positive index value." + case .RedundantLoss: + return "Call to loss API is redundant." + } + } +} + +/// Layer with a 2D shape neural structure and weights. +public class VQ2D: LayerOutput2D, LayerWeightInit +{ + /// The number of vector approximations. + public let K: Int + + /// Coefficient for commitment. + public var beta: Double = 1.0 + + /// + /// Indices of maximal elements. + /// Shape ~ (batch, height, width). + /// + public var indices: MetalBuffer! = nil + + /// + /// Grid of weights. + /// Shape ~ (K, nbChannels). + /// + var _wArrays: WeightGrids! = nil + + /// + /// Buffer of weights. + /// Shape ~ (K, nbChannels). + /// + var _wBuffers: IWeightBuffers! = nil + + /// + /// Buffer of gradients per sample for biases. + /// Shape ~ (batch, K, nbChannels). + /// + var _wDeltaWeights: MetalPrivateBuffer! = nil + + /// Whether to compute weights' gradients or not. + public var computeDeltaWeights: Bool = true + + /// Whether gradients of weights must be accumulated or not. + public var accumulateDeltaWeights: Bool = false + + /// Cache for weights before calling `initKernel` API. + var _weightsList = [Float]() + + /// Weights in the CPU execution context. + public var weightsCPU: [Float] + { + get { + if _wArrays == nil + { + return _weightsList + } + + var weightsTmp = [Float]() + for k in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = VQ2D( + layerPrev: layerPrev, K: K, params: params + ) + layer.coeff = coeff + layer.beta = beta + + if inPlace + { + layer._wArrays = _wArrays + layer._wBuffers = _wBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + _wArrays?.reset() + indices = nil + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + + indices = nil + _wDeltaWeights = nil + _wBuffers?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + public func initWeightsCPU() + { + if _weightsList.count == 0 + { + _weightsList = generateWeightsList() + } + + _wArrays = WeightGrids(width: nbChannels, height: K) + + for k in 0..( + batchSize * height * width, + deviceID: deviceID + ) + } + } + + /// + /// Initialize state resources in the GPU execution context. + /// + /// We initialize the neurons' forward state. + /// We initialize the weights and biases' delta. + /// + public override func checkStateForwardGPU(batchSize: Int) throws + { + try super.checkStateForwardGPU(batchSize: batchSize) + + if computeDeltaWeights && + GrAI.Gradient.sample && _wDeltaWeights == nil + { + _wDeltaWeights = MetalPrivateBuffer( + batchSize * K * nbChannels, deviceID: deviceID + ) + } + + if indices == nil + { + indices = MetalPrivateBuffer( + batchSize * height * width, + deviceID: deviceID + ) + } + } + + /// + /// Check and setup ground truth in the CPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// + public override func checkGroundTruthCPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + fatalError("Not implemented.") + } + + /// + /// Setup groundTruth state in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// - format: The data format. + /// + public override func checkGroundTruthGPU( + _ groundTruth: [T], + batchSize: Int, + nbChannels: Int, height: Int, width: Int, + format: ImageFormat) throws + { + fatalError("Not implemented.") + } + + /// + /// Check and setup ground truth in the GPU execution context. + /// + /// Throw an error if data size is incoherent. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - batchSize: The batch size of data. + /// - nbChannels: Number of channels. + /// - height: Height of each channel. + /// - width: Width of each channel. + /// + public override func checkGroundTruthGPU( + _ groundTruth: MetalBuffer, + batchSize: Int, + nbChannels: Int, height: Int, width: Int) throws + { + fatalError("Not implemented.") + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + fatalError("Not implemented.") + } + + /// + /// Apply the forward pass of the Gradient Checking in GPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCGPU() throws + { + fatalError("Not implemented.") + } + + /// + /// Apply the forward pass in the CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let neuronsPrev = layerPrev.neurons + let indicesPtr = (indices as! MetalSharedBuffer).buffer + + for elem in 0..).buffer + + for elem in 0..).buffer + + if !accumulateDeltaWeights + { + for k in 0..() -> T + { + var losses = [T](repeating: 0.0, count: batchSize) + + if let layerPrev = self.layerPrev as? Layer2D + { + let neuronsPrev = layerPrev.neurons + + for elem in 0..() throws -> T + { + try checkLossGPU(batchSize: batchSize) + + let layerPrev = self.layerPrev as! Layer2D + + let pNbChannels: [UInt32] = [UInt32(nbChannels)] + let pDimensions: [UInt32] = [UInt32(width), UInt32(height)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + + let command = MetalKernel.get.createCommand( + "vq2DLoss", deviceID: deviceID + ) + command.setBuffer(layerPrev.outs.metal, atIndex: 0) + command.setBuffer(outs.metal, atIndex: 1) + command.setBytes(pNbChannels, atIndex: 2) + command.setBytes(pDimensions, atIndex: 3) + command.setBytes(pNbBatch, atIndex: 4) + command.setBuffer(loss.metal, atIndex: 5) + + command.dispatchThreads(batchSize) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for i in 0.. [IWeightArrays] + { + return [_wArrays] + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + return [_wBuffers] + } +} diff --git a/Sources/GrAIdient/LayerSeq/ActivationSeq.swift b/Sources/GrAIdient/LayerSeq/ActivationSeq.swift new file mode 100644 index 00000000..de998d70 --- /dev/null +++ b/Sources/GrAIdient/LayerSeq/ActivationSeq.swift @@ -0,0 +1,344 @@ +// +// ActivationSeq.swift +// GrAIdient +// +// Created by Jean-François Reboud on 21/02/2023. +// + +/// Layer with a sequential shape neural structure and an activation function. +public class ActivationSeq: LayerSeq +{ + /// The activation function. + let _activation: ActivationFunction? + + /// + /// Pre output buffer (result of the forward pass before applying activation) + /// used in the GPU execution context. + /// Shape ~ (batch, nbNeurons). + /// + var _tmp: MetalPrivateBuffer! = nil + + /// Get coefficient (depending on activation function) to apply during the weights initialization. + public var coeffInitWeights: Float + { + get { + if let activation = _activation + { + return activation.coeffInitWeights + } + return 1.0 + } + } + + private enum Keys: String, CodingKey + { + case activation + } + + /// + /// Create a layer with a sequential shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - activation: The activation function. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: LayerSeq, + activation: String, + params: GrAI.Model.Params) + { + _activation = GrAI.Model.Activation.build(activation) + + super.init(layerPrev: layerPrev, + sequence: layerPrev.sequence, + nbNeurons: layerPrev.nbNeurons, + params: params) + } + + /// + /// Create a layer with a 1D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - nbNeurons: Number of neurons. + /// - activation: The activation function. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer?, + sequence: Int, nbNeurons: Int, activation: String?, + params: GrAI.Model.Params) + { + if let activationStr = activation + { + _activation = GrAI.Model.Activation.build(activationStr) + } + else + { + _activation = nil + } + + super.init(layerPrev: layerPrev, + sequence: sequence, + nbNeurons: nbNeurons, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let container = try decoder.container(keyedBy: Keys.self) + _activation = + try container.decodeIfPresent(ActivationContainer.self, + forKey: .activation)?.activation + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + if let activation = _activation + { + try container.encode(ActivationContainer(activation), + forKey: Keys.activation) + } + try super.encode(to: encoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = ActivationSeq( + layerPrev: layerPrev, + activation: _activation!.name, + params: params + ) + return layer + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// State resources are the resources that are dependent on the batch size. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + _tmp = nil + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try _forwardGC() + _activation!.forwardGC(self) + } + + /// + /// Apply the forward pass (until the activation function) of the Gradient Checking. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + private func _forwardGC() throws + { + if let layerPrev = self.layerPrev as? LayerSeq + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for seq in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = AvgPoolSeq(layerPrev: layerPrev, params: params) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? LayerSeq + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for depth in 0.. ([Layer], [Int]) + { + var layersBranches = [Layer?]() + for layer in _layersPrev + { + layersBranches.append(layer) + } + + let layersEqual = + { + () -> Bool in + let firstLayer = layersBranches.first! + for layer in layersBranches + { + if layer !== firstLayer + { + return false + } + } + return true + } + + var layersIndex = [Int]() + var layers = [Layer]() + while !layersEqual() + { + var idMax = -1 + var indexMax = -1 + + for (index, layer) in layersBranches.enumerated() + { + if let layerTmp = layer + { + let id = layerTmp.id + if id > idMax + { + idMax = id + indexMax = index + } + } + } + if indexMax < 0 + { + break + } + + let layerMax = layersBranches[indexMax]! + layersBranches[indexMax] = layerMax.layerPrev + + layersIndex.append(indexMax) + layers.append(layerMax) + } + + return (layers, layersIndex) + } + + /// + /// Get every layers (a "graph") between the very first of the `Model` and this. + /// + /// - Parameter layerPrev: The different layers found in the "graph". + /// + public override func getGraph(_ layers: inout [Layer]) + { + layers.append(self) + + let layersMerged = _getMergedGraph().0 + layers += layersMerged + + layersMerged.last?.layerPrev?.getGraph(&layers) + } + + /// + /// Get every layers (a "graph") between the very first of the `Model` and this. + /// + /// The main difficulty with a `LayerMerge` is that we must take into account the origin of the + /// weight modifications for estimating their gradient during the Gradient Checking. + /// When we look at the "graph" of a `LayerMerge` we must consider the last common ancestor + /// before the fork. + /// The weights originating before the fork should only undergo a "simple forward" from the + /// layers that appear after the fork. + /// But the weights modifications that pop after a fork should have a particular behavior as they + /// are populating a new weight modification that is related to one precise branch. + /// + /// - Returns: (Number of weight modifications that occur before the fork, + /// Index of the different layers after the fork, + /// Number of weight modifications associated with the different layers after the fork). + /// + public func getMergedGraph() -> (nbSameElems: Int, + layersIndex: [Int], + nbElems: [Int]) + { + var (layersMerged, layersIndex) = _getMergedGraph() + + var nbSameElems = 0 + if let commonAncestor = layersMerged.last!.layerPrev + { + nbSameElems = commonAncestor.nbGC + } + + layersMerged = layersMerged.reversed() + layersIndex = layersIndex.reversed() + + var nbElems = [Int]() + var nbLastElems = [Int](repeating: nbSameElems, + count: _layersPrev.count) + for (index, layer) in zip(layersIndex, layersMerged) + { + let nbElemsTmp = layer.nbGC + let nbDiffElems = nbElemsTmp - nbLastElems[index] + + nbLastElems[index] += nbDiffElems + nbElems.append(nbDiffElems) + } + + return (nbSameElems: nbSameElems, + layersIndex: layersIndex, + nbElems: nbElems) + } +} diff --git a/Sources/GrAIdient/LayerSeq/Base/LayerSeq.swift b/Sources/GrAIdient/LayerSeq/Base/LayerSeq.swift new file mode 100644 index 00000000..0a79d55d --- /dev/null +++ b/Sources/GrAIdient/LayerSeq/Base/LayerSeq.swift @@ -0,0 +1,180 @@ +// +// LayerSeq.swift +// GrAIdient +// +// Created by Jean-François Reboud on 20/02/2023. +// + +/// Layer with a sequential shape neural structure. +open class LayerSeq: Layer +{ + /// Neural structure used in the CPU execution context. + public internal(set) var neurons: GridNeurons! = nil + + /// + /// Output buffer (result of the forward pass) used in the GPU execution context. + /// Shape ~ (batch, seq, nbNeurons). + /// + public var outs: MetalPrivateBuffer! = nil + /// + /// Gradient buffer (result of the backward pass) used in the GPU execution context. + /// Shape ~ (batch, seq, nbNeurons). + /// + public var delta: MetalPrivateBuffer! = nil + + /// Length of the sequence. + public let sequence: Int + /// Number of neurons. + public let nbNeurons: Int + + /// Number of different weigths for which we are estimating the gradient during Gradient Checking. + public override var nbGC: Int + { + get { + return neurons.get(0, 0)!.nbGC + } + } + + private enum Keys: String, CodingKey + { + case sequence + case nbNeurons + } + + /// + /// Create a layer with a 1D shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - sequence: Length of the sequence. + /// - nbNeurons: Number of neurons. + /// - params: Contextual parameters linking to the model. + /// + public init(layerPrev: Layer?, + sequence: Int, + nbNeurons: Int, + params: GrAI.Model.Params) + { + self.sequence = sequence + self.nbNeurons = nbNeurons + super.init(layerPrev: layerPrev, params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let container = try decoder.container(keyedBy: Keys.self) + sequence = try container.decode(Int.self, forKey: .sequence) + nbNeurons = try container.decode(Int.self, forKey: .nbNeurons) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + open override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + try container.encode(sequence, forKey: .sequence) + try container.encode(nbNeurons, forKey: .nbNeurons) + try super.encode(to: encoder) + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We clean the neurons' state (forward and backward). + /// + open override func resetKernelCPU() + { + super.resetKernelCPU() + neurons = nil + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We clean the neurons' state (forward and backward). + /// + open override func resetKernelGPU() + { + super.resetKernelGPU() + outs = nil + delta = nil + } + + /// + /// Initialize state resources in the CPU execution context. + /// + /// We initialize the neurons' state (forward and backward). + /// + public func checkStateCPU(batchSize: Int) throws + { + if neurons == nil + { + neurons = GridNeurons(width: nbNeurons, height: sequence) + for neuron in neurons.all + { + neuron.initBatch(batchSize) + } + } + else if batchSize <= 0 || batchSize > neurons.get(0)!.v.count + { + throw LayerError.BatchSize + } + } + + /// + /// Initialize state resources in the GPU execution context. + /// + /// We initialize the neurons' forward state. + /// + public func checkStateForwardGPU(batchSize: Int) throws + { + if outs == nil + { + outs = MetalPrivateBuffer( + batchSize * sequence * nbNeurons, deviceID: deviceID + ) + } + else if batchSize <= 0 || batchSize > outs.nbElems / nbNeurons + { + throw LayerError.BatchSize + } + } + + /// + /// Initialize state resources in the GPU execution context. + /// + /// We initialize the neurons' backward state. + /// + public func checkStateBackwardGPU(batchSize: Int) throws + { + if delta == nil + { + delta = MetalPrivateBuffer( + batchSize * sequence * nbNeurons, deviceID: deviceID + ) + } + else if batchSize <= 0 || + batchSize > delta.nbElems / (sequence * nbNeurons) + { + throw LayerError.BatchSize + } + } +} diff --git a/Sources/GrAIdient/LayerSeq/ConcatSeq.swift b/Sources/GrAIdient/LayerSeq/ConcatSeq.swift new file mode 100644 index 00000000..fae570e4 --- /dev/null +++ b/Sources/GrAIdient/LayerSeq/ConcatSeq.swift @@ -0,0 +1,848 @@ +// +// ConcatSeq.swift +// GrAIdient +// +// Created by Jean-François Reboud on 05/03/2023. +// + +/// +/// Layer with a sequential shape neural structure. +/// +/// This layer merges multiple sequential layers, +/// concatenating the neurons in the sequential dimension together. +/// +public class Concat1Seq: LayerMergeSeq +{ + /// + /// Create a layer with a sequential shape neural structure. + /// + /// - Parameters: + /// - layersPrev: List of previous layers that have been queued to the model. + /// - params: Contextual parameters linking to the model. + /// + public init(layersPrev: [LayerSeq], params: GrAI.Model.Params) throws + { + let layer0 = layersPrev[0] + let nbNeurons = layer0.nbNeurons + for layerPrev in layersPrev + { + if layerPrev.nbNeurons != nbNeurons + { + throw LayerError.Init(message: "Layer structure error.") + } + } + + var sequence = 0 + for layer in layersPrev + { + sequence += layer.sequence + } + super.init(layersPrev: layersPrev, + sequence: sequence, + nbNeurons: nbNeurons, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + try super.init(from: decoder) + } + + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + var layersPrev = [LayerSeq]() + for idPrev in _idsPrev + { + layersPrev.append(mapping[idPrev] as! LayerSeq) + } + + let layer = try! Concat1Seq(layersPrev: layersPrev, params: params) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: batchSize) + + let (nbSameElems, layersIndex, nbElems) = getMergedGraph() + + var nbGC = nbSameElems + for nbElemsTmp in nbElems + { + nbGC += nbElemsTmp + } + + for seq in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + var layersPrev = [LayerSeq]() + for idPrev in _idsPrev + { + layersPrev.append(mapping[idPrev] as! LayerSeq) + } + + let layer = Concat2Seq(layersPrev: layersPrev, params: params) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: batchSize) + + let (nbSameElems, layersIndex, nbElems) = getMergedGraph() + + var nbGC = nbSameElems + for nbElemsTmp in nbElems + { + nbGC += nbElemsTmp + } + + for seq in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = Constant12Seq( + sequence: sequence, + nbNeurons: nbNeurons, + params: params + ) + if inPlace + { + layer._wArrays = _wArrays + layer._wBuffers = _wBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + _wArrays?.reset() + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + _wBuffers?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + public func initWeightsCPU() + { + _wArrays = WeightGrids(width: nbNeurons, height: sequence) + + if _weightsList.count == 0 + { + for seq in 0.. [IWeightArrays] + { + return [_wArrays] + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + return [_wBuffers] + } +} + +/// +/// Layer with a sequential shape neural structure and weights. +/// +/// The weights only target the "neural" dimension here. +/// +public class Constant2Seq: LayerSeq, LayerUpdate +{ + /// + /// Grid of weights. + /// Shape ~ (nbNeurons,). + /// + var _wArrays: WeightArrays! = nil + + /// + /// Buffer of weights. + /// Shape ~ (nbNeurons,). + /// + var _wBuffers: IWeightBuffers! = nil + + /// + /// Buffer of gradients per sample for biases. + /// Shape ~ (batch, sequence, nbNeurons). + /// + var _wDeltaWeights: MetalPrivateBuffer! = nil + + /// Whether to compute weights' gradients or not. + public var computeDeltaWeights: Bool = true + + /// Whether gradients of weights must be accumulated or not. + public var accumulateDeltaWeights: Bool = false + + /// Cache for weights before calling `initKernel` API. + var _weightsList = [Float]() + + /// Weights in the CPU execution context. + public var weightsCPU: [Float] + { + get { + if _wArrays == nil + { + return _weightsList + } + + var weightsTmp = [Float]() + for depth in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = Constant2Seq( + sequence: sequence, + nbNeurons: nbNeurons, + params: params + ) + if inPlace + { + layer._wArrays = _wArrays + layer._wBuffers = _wBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + _wArrays?.reset() + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + + _wDeltaWeights = nil + _wBuffers?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + public func initWeightsCPU() + { + _wArrays = WeightArrays(nbNeurons) + + if _weightsList.count == 0 + { + for depth in 0..( + batchSize * sequence * nbNeurons, deviceID: deviceID + ) + } + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: batchSize) + + let newGC = 2 * nbLearnedGC + for seq in 0.. [IWeightArrays] + { + return [_wArrays] + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + return [_wBuffers] + } + + /// + /// Get the weights' gradients in the GPU execution context. + /// + /// Throw an error when layer has not been updated through backward pass or + /// when gradients per sample have not been computed. + /// + /// - Parameter elem: The batch element to retrieve the outputs from. + /// + public func getDeltaWeightsGPU(elem: Int) throws + -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + if !GrAI.Gradient.sample + { + throw UpdateError.PerSample + } + + var deltaWeights = [T]() + MetalKernel.get.download([_wDeltaWeights]) + let deltaWeightsPtr = _wDeltaWeights.shared.buffer + + for depth in 0..() throws -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + + var deltaWeights = [T]() + for depth in 0..() throws -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + + var deltaWeights = [T]() + MetalKernel.get.download([_wBuffers.g_p!]) + let deltaWeightsPtr = _wBuffers.g_p!.shared.buffer + + for i in 0..<_wBuffers.nbElems + { + deltaWeights.append(T(deltaWeightsPtr[i])) + } + return deltaWeights + } +} diff --git a/Sources/GrAIdient/LayerSeq/FullyConnectedPatch.swift b/Sources/GrAIdient/LayerSeq/FullyConnectedPatch.swift new file mode 100644 index 00000000..9ed2b6ce --- /dev/null +++ b/Sources/GrAIdient/LayerSeq/FullyConnectedPatch.swift @@ -0,0 +1,1421 @@ +// +// FullyConnectedPatch.swift +// GrAIdient +// +// Created by Jean-François Reboud on 21/02/2023. +// + +import Foundation + +/// +/// Layer with a sequential shape neural structure, weights and biases and an activation function. +/// +/// This layer transforms a 2D layer into a sequential layer, applying fully connected operation +/// on patches of the 2D layer. +/// +public class FullyConnectedPatch: ActivationSeq, + LayerWithActivation, LayerWeightInit +{ + /// Size (height, width) of a patch. + let _patch: Int + + /// + /// Grid of weights. + /// Shape ~ (nbNeurons, nbNeuronsPrev x patch x patch). + /// + var _wArrays: WeightGrids! = nil + /// + /// Array of biases. + /// Shape ~ (nbNeurons,). + /// + var _bArrays: WeightArrays! = nil + + /// + /// Buffer of weights. + /// Shape ~ (nbNeurons, nbNeuronsPrev x patch x patch). + /// + var _wBuffers: IWeightBuffers! = nil + /// + /// Buffer of biases. + /// Shape ~ (nbNeurons,). + /// + var _bBuffers: IWeightBuffers! = nil + + /// + /// Buffer of gradients per sample for weights. + /// Shape ~ (batch, nbNeurons, nbNeuronsPrev x patch x patch). + /// + var _wDeltaWeights: MetalPrivateBuffer! = nil + /// + /// Buffer of gradients per sample for biases. + /// Shape ~ (batch, nbNeurons). + /// + var _bDeltaWeights: MetalPrivateBuffer! = nil + + /// Whether to compute weights' gradients or not. + public var computeDeltaWeights: Bool = true + + /// Whether gradients of weights must be accumulated or not. + public var accumulateDeltaWeights: Bool = false + + /// Height of the weight's grid shape. + public let weightHeight: Int + /// Width of the weight's grid shape. + public let weightWidth: Int + + /// Whether to update biases or not. + var _updateBiases: Bool = true + + /// Cache for weights before calling `initKernel` API. + var _weightsList = [Float]() + + /// Weights in the CPU execution context. + public var weightsCPU: [Float] + { + get { + if _wArrays == nil + { + return _weightsList + } + + var weightsTmp = [Float]() + for i in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = try! FullyConnectedPatch( + layerPrev: layerPrev, + patch: _patch, + nbNeurons: nbNeurons, + activation: _activation?.name, + biases: _updateBiases, + params: params + ) + if inPlace + { + layer._wArrays = _wArrays + layer._bArrays = _bArrays + layer._wBuffers = _wBuffers + layer._bBuffers = _bBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Extract main operation of this layer without the activation part. + /// + /// This API will create a new layer in the same context as this. + /// + /// - Parameter inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new instance of `Layer`. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public func removeActivation(inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = self.layerPrev as! Layer2D + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = try! FullyConnectedPatch( + layerPrev: layerPrev, + patch: _patch, + nbNeurons: nbNeurons, + activation: nil, + biases: _updateBiases, + params: params + ) + if inPlace + { + layer._wArrays = _wArrays + layer._bArrays = _bArrays + layer._wBuffers = _wBuffers + layer._bBuffers = _bBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Extract main operation of this layer without the activation part. + /// + /// - Parameter params: Contextual parameters linking to the model. + /// + /// - Returns: A new layer. + /// + public func removeActivation(params: GrAI.Model.Params) -> Layer + { + let layerPrev = self.layerPrev as! Layer2D + let layer = try! FullyConnectedPatch( + layerPrev: layerPrev, + patch: _patch, + nbNeurons: nbNeurons, + activation: nil, + biases: _updateBiases, + params: params + ) + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + + _wArrays?.reset() + _bArrays?.reset() + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + + _wDeltaWeights = nil + _bDeltaWeights = nil + + _wBuffers?.reset() + _bBuffers?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + public func initWeightsCPU() + { + if _weightsList.count == 0 + { + _weightsList = generateWeightsList() + _weightsList += [Float](repeating: 0.0, count: weightHeight) + } + + _wArrays = WeightGrids(width: weightWidth, height: weightHeight) + _bArrays = WeightArrays(weightHeight) + + for i in 0..( + batchSize * sequence * nbNeurons * weightWidth, + deviceID: deviceID + ) + + if _updateBiases + { + _bDeltaWeights = MetalPrivateBuffer( + batchSize * sequence * nbNeurons, deviceID: deviceID + ) + } + } + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try _forwardGCCPU() + _activation?.forwardGC(self) + } + + private func _forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? Layer2D + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + let newGC = nbGC + 2 * nbLearnedGC + + for seq in 0.. [IWeightArrays] + { + var weights = [IWeightArrays]() + weights.append(_wArrays) + if _updateBiases + { + weights.append(_bArrays) + } + return weights + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + var weights = [IWeightBuffers]() + weights.append(_wBuffers) + if _updateBiases + { + weights.append(_bBuffers) + } + return weights + } + + /// + /// Get the weights' gradients in the GPU execution context. + /// + /// Throw an error when layer has not been updated through backward pass or + /// when gradients per sample have not been computed. + /// + /// - Parameter elem: The batch element to retrieve the outputs from. + /// + public func getDeltaWeightsGPU(elem: Int) throws + -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + if !GrAI.Gradient.sample + { + throw UpdateError.PerSample + } + + var deltaWeights = [T]() + MetalKernel.get.download([_wDeltaWeights]) + var deltaWeightsPtr = _wDeltaWeights.shared.buffer + + let offsetStart = elem * nbNeurons * weightWidth + for depth in 0..() throws -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + + var deltaWeights = [T]() + for depth in 0..() throws -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + + var deltaWeights = [T]() + MetalKernel.get.download([_wBuffers.g_p!]) + var deltaWeightsPtr = _wBuffers.g_p!.shared.buffer + + for i in 0..<_wBuffers.nbElems + { + deltaWeights.append(T(deltaWeightsPtr[i])) + } + if _updateBiases + { + MetalKernel.get.download([_bBuffers.g_p!]) + deltaWeightsPtr = _bBuffers.g_p!.shared.buffer + + for i in 0..<_bBuffers.nbElems + { + deltaWeights.append(T(deltaWeightsPtr[i])) + } + } + return deltaWeights + } +} diff --git a/Sources/GrAIdient/LayerSeq/FullyConnectedSeq.swift b/Sources/GrAIdient/LayerSeq/FullyConnectedSeq.swift new file mode 100644 index 00000000..2c6d71cc --- /dev/null +++ b/Sources/GrAIdient/LayerSeq/FullyConnectedSeq.swift @@ -0,0 +1,1264 @@ +// +// FullyConnectedSeq.swift +// GrAIdient +// +// Created by Jean-François Reboud on 08/03/2023. +// + +import Foundation + +/// Layer with a sequential shape neural structure, weights and biases and an activation function. +public class FullyConnectedSeq: ActivationSeq, + LayerWithActivation, LayerWeightInit +{ + /// + /// Grid of weights. + /// Shape ~ (nbNeurons, nbNeuronsPrev). + /// + var _wArrays: WeightGrids! = nil + /// + /// Array of biases. + /// Shape ~ (nbNeurons,). + /// + var _bArrays: WeightArrays! = nil + + /// + /// Buffer of weights. + /// Shape ~ (nbNeurons, nbNeuronsPrev). + /// + var _wBuffers: IWeightBuffers! = nil + /// + /// Buffer of biases. + /// Shape ~ (nbNeurons,). + /// + var _bBuffers: IWeightBuffers! = nil + + /// + /// Buffer of gradients per sample for weights. + /// Shape ~ (batch, nbNeurons, nbNeuronsPrev). + /// + var _wDeltaWeights: MetalPrivateBuffer! = nil + /// + /// Buffer of gradients per sample for biases. + /// Shape ~ (batch, nbNeurons). + /// + var _bDeltaWeights: MetalPrivateBuffer! = nil + + /// Whether to compute weights' gradients or not. + public var computeDeltaWeights: Bool = true + + /// Whether gradients of weights must be accumulated or not. + public var accumulateDeltaWeights: Bool = false + + /// Height of the weight's grid shape. + public let weightHeight: Int + /// Width of the weight's grid shape. + public let weightWidth: Int + + /// Whether to update biases or not. + var _updateBiases: Bool = true + + /// Cache for weights before calling `initKernel` API. + var _weightsList = [Float]() + + /// Weights in the CPU execution context. + public var weightsCPU: [Float] + { + get { + if _wArrays == nil + { + return _weightsList + } + + var weightsTmp = [Float]() + for i in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = FullyConnectedSeq( + layerPrev: layerPrev, + nbNeurons: nbNeurons, + activation: _activation?.name, + biases: _updateBiases, + params: params + ) + if inPlace + { + layer._wArrays = _wArrays + layer._bArrays = _bArrays + layer._wBuffers = _wBuffers + layer._bBuffers = _bBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Extract main operation of this layer without the activation part. + /// + /// This API will create a new layer in the same context as this. + /// + /// - Parameter inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new instance of `Layer`. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public func removeActivation(inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = self.layerPrev as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = FullyConnectedSeq( + layerPrev: layerPrev, + nbNeurons: nbNeurons, + activation: nil, + biases: _updateBiases, + params: params + ) + if inPlace + { + layer._wArrays = _wArrays + layer._bArrays = _bArrays + layer._wBuffers = _wBuffers + layer._bBuffers = _bBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Extract main operation of this layer without the activation part. + /// + /// - Parameter params: Contextual parameters linking to the model. + /// + /// - Returns: A new layer. + /// + public func removeActivation(params: GrAI.Model.Params) -> Layer + { + let layerPrev = self.layerPrev as! LayerSeq + let layer = FullyConnectedSeq( + layerPrev: layerPrev, + nbNeurons: nbNeurons, + activation: nil, + biases: _updateBiases, + params: params + ) + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + + _wArrays?.reset() + _bArrays?.reset() + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + + _wDeltaWeights = nil + _bDeltaWeights = nil + + _wBuffers?.reset() + _bBuffers?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + public func initWeightsCPU() + { + if _weightsList.count == 0 + { + _weightsList = generateWeightsList() + _weightsList += [Float](repeating: 0.0, count: weightHeight) + } + + _wArrays = WeightGrids(width: weightWidth, height: weightHeight) + _bArrays = WeightArrays(weightHeight) + + for i in 0..( + batchSize * sequence * nbNeurons * weightWidth, + deviceID: deviceID + ) + + if _updateBiases + { + _bDeltaWeights = MetalPrivateBuffer( + batchSize * sequence * nbNeurons, deviceID: deviceID + ) + } + } + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try _forwardGCCPU() + _activation?.forwardGC(self) + } + + private func _forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? LayerSeq + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + let newGC = nbGC + 2 * nbLearnedGC + + for seq in 0.. [IWeightArrays] + { + var weights = [IWeightArrays]() + weights.append(_wArrays) + if _updateBiases + { + weights.append(_bArrays) + } + return weights + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + var weights = [IWeightBuffers]() + weights.append(_wBuffers) + if _updateBiases + { + weights.append(_bBuffers) + } + return weights + } + + /// + /// Get the weights' gradients in the GPU execution context. + /// + /// Throw an error when layer has not been updated through backward pass or + /// when gradients per sample have not been computed. + /// + /// - Parameter elem: The batch element to retrieve the outputs from. + /// + public func getDeltaWeightsGPU(elem: Int) throws + -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + if !GrAI.Gradient.sample + { + throw UpdateError.PerSample + } + + var deltaWeights = [T]() + MetalKernel.get.download([_wDeltaWeights]) + var deltaWeightsPtr = _wDeltaWeights.shared.buffer + + let offsetStart = elem * nbNeurons * weightWidth + for depth in 0..() throws -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + + var deltaWeights = [T]() + for depth in 0..() throws -> [T] + { + if dirty + { + throw UpdateError.Dirty + } + + var deltaWeights = [T]() + MetalKernel.get.download([_wBuffers.g_p!]) + var deltaWeightsPtr = _wBuffers.g_p!.shared.buffer + + for i in 0..<_wBuffers.nbElems + { + deltaWeights.append(T(deltaWeightsPtr[i])) + } + if _updateBiases + { + MetalKernel.get.download([_bBuffers.g_p!]) + deltaWeightsPtr = _bBuffers.g_p!.shared.buffer + + for i in 0..<_bBuffers.nbElems + { + deltaWeights.append(T(deltaWeightsPtr[i])) + } + } + return deltaWeights + } +} diff --git a/Sources/GrAIdient/LayerSeq/LayerNormSeq.swift b/Sources/GrAIdient/LayerSeq/LayerNormSeq.swift new file mode 100644 index 00000000..64333c72 --- /dev/null +++ b/Sources/GrAIdient/LayerSeq/LayerNormSeq.swift @@ -0,0 +1,730 @@ +// +// LayerNormSeq.swift +// GrAIdient +// +// Created by Jean-François Reboud on 08/03/2023. +// + +/// Layer with a sequential shape neural structure, an activation function and one layer normalization unit. +public class LayerNormSeq: ActivationSeq, LayerUpdate, LayerWithActivation +{ + /// Instance normalization by default or instance normalization in the CPU execution context. + var _norm: LayerWeightsNormalization? = nil + /// Instance normalization in the GPU execution context. + var _normGPU: LayerNormalizationGPU? = nil + + /// Whether to compute weights' gradients or not. + public var computeDeltaWeights: Bool = true + + /// Whether gradients of weights must be accumulated or not. + public var accumulateDeltaWeights: Bool = false + + /// Weights in the CPU execution context. + public var weightsCPU: [Float] + { + get { + var weightsTmp = [Float]() + if let norm = _norm + { + weightsTmp += norm.weights + } + return weightsTmp + } + set { + if let norm = _norm + { + norm.weights = newValue + } + } + } + + /// Weights in the GPU execution context. + public var weightsGPU: [Float] + { + get { + var weightsTmp = [Float]() + if let norm = _normGPU + { + weightsTmp += norm.weights + } + else if let norm = _norm + { + weightsTmp += norm.weights + } + return weightsTmp + } + set { + if let norm = _normGPU + { + norm.weights = newValue + } + else if let norm = _norm + { + norm.weights = newValue + } + } + } + + /// Get instance normalization in the CPU execution context. + var norm: LayerNormalization? + { + get { + return _norm as? LayerNormalization + } + } + + /// Number of new weights due to this layer, estimated during the Gradient Checking. + var nbLearnedGC: Int + { + get { + return 2 * nbNeurons + } + } + + private enum Keys: String, CodingKey + { + case norm + } + + /// + /// Create a layer with a sequential shape neural structure. + /// + /// - Parameters: + /// - layerPrev: Previous layer that has been queued to the model. + /// - activation: The activation function. + /// - params: Contextual parameters linking to the model. + /// + public override init(layerPrev: LayerSeq, activation: String?, + params: GrAI.Model.Params) + { + super.init(layerPrev: layerPrev, + sequence: layerPrev.sequence, + nbNeurons: layerPrev.nbNeurons, + activation: activation, + params: params) + + _norm = LayerWeightsNormalization(self) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let values = try decoder.container(keyedBy: Keys.self) + _norm = try values.decodeIfPresent( + LayerWeightsNormalization.self, forKey: .norm + ) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + if let norm = _normGPU + { + try container.encode(norm, forKey: Keys.norm) + } + else if let norm = _norm + { + try container.encode(norm, forKey: Keys.norm) + } + try super.encode(to: encoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = LayerNormSeq( + layerPrev: layerPrev, + activation: _activation?.name, + params: params + ) + if inPlace + { + layer._norm = _norm + layer._normGPU = _normGPU + } + else + { + // only one of them should be cloned + if let norm = _normGPU + { + layer._norm = norm.clone() + } + else if let norm = _norm + { + layer._norm = norm.clone() + } + } + return layer + } + + /// + /// Extract main operation of this layer without the activation part. + /// + /// This API will create a new layer in the same context as this. + /// + /// - Parameter inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new instance of `Layer`. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public func removeActivation(inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = self.layerPrev as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = LayerNormSeq( + layerPrev: layerPrev, + activation: nil, + params: params + ) + if inPlace + { + layer._norm = _norm + layer._normGPU = _normGPU + } + else + { + // only one of them should be cloned + if let norm = _normGPU + { + layer._norm = norm.clone() + } + else if let norm = _norm + { + layer._norm = norm.clone() + } + } + + return layer + } + + /// + /// Extract main operation of this layer without the activation part. + /// + /// - Parameter params: Contextual parameters linking to the model. + /// + /// - Returns: A new layer. + /// + public func removeActivation(params: GrAI.Model.Params) -> Layer + { + let layerPrev = self.layerPrev as! LayerSeq + let layer = LayerNormSeq( + layerPrev: layerPrev, + activation: nil, + params: params + ) + // only one of them should be cloned + if let norm = _normGPU + { + layer._norm = norm.clone() + } + else if let norm = _norm + { + layer._norm = norm.clone() + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We reset batch normalization. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + norm?.resetKernel() + } + /// + /// Clean state resources in the GPU execution context. + /// + /// We reset batch normalization. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + _normGPU?.resetKernel() + } + + /// + /// Initialize hard resources in the CPU execution context. + /// + /// We initialize batch normalization. + /// + public override func initKernelCPU() + { + super.initKernelCPU() + + if let norm = _normGPU + { + _norm = LayerNormalization(norm: norm) + } + else if let norm = _norm + { + _norm = LayerNormalization(norm: norm) + } + + if !GrAI.Loop.gradientChecking + { + _normGPU = nil + } + } + + /// + /// Initialize hard resources in the GPU execution context. + /// + /// We initialize batch normalization. + /// + public override func initKernelGPU() + { + super.initKernelGPU() + + if let norm = _normGPU + { + _normGPU = LayerNormalizationGPU(norm: norm) + } + else if let norm = _norm + { + _normGPU = LayerNormalizationGPU(norm: norm) + } + _normGPU?.initKernel(deviceID: deviceID) + + if !GrAI.Loop.gradientChecking + { + _norm = nil + } + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// We initialize batch normalization's weights. + /// + public func initWeightsCPU() + { + norm?.initWeights() + } + /// + /// Initialize weights in the GPU execution context. + /// + /// We initialize batch normalization's weights. + /// + public func initWeightsGPU() + { + _normGPU?.initWeights() + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try _forwardGCCPU() + norm!.forwardGC(self) + _activation?.forwardGC(self) + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + private func _forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? LayerSeq + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + let newGC = nbGC + 2 * nbLearnedGC + for seq in 0.. [IWeightArrays] + { + var weights = [IWeightArrays]() + if let norm = self.norm + { + weights += norm.collectWeights() + } + return weights + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + return _normGPU!.collectWeights() + } + + /// + /// Get the outputs of Gradient Checking (result of the forward pass) in the CPU execution context. + /// + /// - Parameters: + /// - batch: Index of sample in the mini batch. + /// - seq: Index of the sequence. + /// - elem: Weight estimation index during the Gradient Checking. + /// - Returns: The outputs. + /// + func getOutsGC(batch: Int, seq: Int, elem: Int) -> [Double] + { + var outs = [Double](repeating: 0.0, count: nbNeurons) + for depth in 0.. [Double] + { + var outs = [Double](repeating: 0.0, count: nbNeurons) + for depth in 0.. [Double] + { + var delta = [Double](repeating: 0.0, count: nbNeurons) + for depth in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + var layersPrev = [LayerSeq]() + for idPrev in _idsPrev + { + layersPrev.append(mapping[idPrev] as! LayerSeq) + } + + let layer = try! QuerySeq( + query: layersPrev[0], key: layersPrev[1], nbHeads: _nbHeads, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: batchSize) + + let (nbSameElems, layersIndex, nbElems) = getMergedGraph() + + var nbGC = nbSameElems + for nbElemsTmp in nbElems + { + nbGC += nbElemsTmp + } + + for seqQ in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = SelectSeq( + layerPrev: layerPrev, + targetSeq: _targetSeq, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? LayerSeq + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for depth in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = try! SoftmaxSeq( + layerPrev: layerPrev, + nbHeads: _nbHeads, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + if let layerPrev = self.layerPrev as? LayerSeq + { + try checkStateCPU(batchSize: batchSize) + + let nbGC = layerPrev.nbGC + for seq in 0.. cMax + { + cMax = outPrev + } + } + + var sum1 = 0.0 + for j in 0.. cMax + { + cMax = outPrev + } + } + + var sum1 = 0.0 + for j in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + var layersPrev = [LayerSeq]() + for idPrev in _idsPrev + { + layersPrev.append(mapping[idPrev] as! LayerSeq) + } + + let layer = try! SumSeq(layersPrev: layersPrev, params: params) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: batchSize) + + let (nbSameElems, layersIndex, nbElems) = getMergedGraph() + + var nbGC = nbSameElems + for nbElemsTmp in nbElems + { + nbGC += nbElemsTmp + } + + for seq in 0..! = nil + /// + /// Indices of maximal elements. + /// Shape ~ (batch, seq). + /// + public var indices: MetalBuffer! = nil + + /// + /// Grid of weights. + /// Shape ~ (K, nbNeurons). + /// + var _wArrays: WeightGrids! = nil + + /// + /// Buffer of weights. + /// Shape ~ (K, nbNeurons). + /// + var _wBuffers: IWeightBuffers! = nil + + /// + /// Buffer of gradients per sample for biases. + /// Shape ~ (batch, K, nbNeurons). + /// + var _wDeltaWeights: MetalPrivateBuffer! = nil + + /// Whether to compute weights' gradients or not. + public var computeDeltaWeights: Bool = true + + /// Whether gradients of weights must be accumulated or not. + public var accumulateDeltaWeights: Bool = false + + /// Cache for weights before calling `initKernel` API. + var _weightsList = [Float]() + + /// Weights in the CPU execution context. + public var weightsCPU: [Float] + { + get { + if _wArrays == nil + { + return _weightsList + } + + var weightsTmp = [Float]() + for k in 0.., + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let layerPrev = mapping[idPrev] as! LayerSeq + + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + let layer = VQSeq( + layerPrev: layerPrev, K: K, params: params + ) + layer.coeff = coeff + layer.beta = beta + + if inPlace + { + layer._wArrays = _wArrays + layer._wBuffers = _wBuffers + } + else + { + if GrAI.Opti.GPU + { + layer.weightsGPU = weightsGPU + } + else + { + layer.weightsCPU = weightsCPU + } + } + return layer + } + + /// + /// Clean state resources in the CPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelCPU() + { + super.resetKernelCPU() + _wArrays?.reset() + indices = nil + } + + /// + /// Clean state resources in the GPU execution context. + /// + /// We first clean the neurons' state (forward and backward). + /// We do not clean weights and biases but must reset their delta (dependent on batch size) and + /// momentum state. + /// + public override func resetKernelGPU() + { + super.resetKernelGPU() + + indices = nil + _wDeltaWeights = nil + _wBuffers?.reset() + } + + /// + /// Initialize weights in the CPU execution context. + /// + /// Their momentum and delta state are also reset. + /// + public func initWeightsCPU() + { + if _weightsList.count == 0 + { + _weightsList = generateWeightsList() + } + + _wArrays = WeightGrids(width: nbNeurons, height: K) + + for k in 0..( + batchSize * sequence, + deviceID: deviceID + ) + } + } + + /// + /// Initialize state resources in the GPU execution context. + /// + /// We initialize the neurons' forward state. + /// We initialize the weights and biases' delta. + /// + public override func checkStateForwardGPU(batchSize: Int) throws + { + try super.checkStateForwardGPU(batchSize: batchSize) + + if computeDeltaWeights && + GrAI.Gradient.sample && _wDeltaWeights == nil + { + _wDeltaWeights = MetalPrivateBuffer( + batchSize * K * nbNeurons, deviceID: deviceID + ) + } + + if indices == nil + { + indices = MetalPrivateBuffer( + batchSize * sequence, + deviceID: deviceID + ) + } + } + + /// + /// Setup loss state in the GPU execution context. + /// + /// Throw an error if batch size or ground truth are incoherent. + /// + /// - Parameter batchSize: The batch size of data. + /// + public func checkLossGPU(batchSize: Int) throws + { + if loss == nil + { + loss = MetalSharedBuffer(batchSize, deviceID: deviceID) + } + else if batchSize <= 0 || batchSize > loss.nbElems + { + throw LayerError.BatchSize + } + } + + /// + /// Apply the forward pass in the CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardCPU() throws + { + if let layerPrev = self.layerPrev as? LayerSeq + { + try checkStateCPU(batchSize: batchSize) + + let neuronsPrev = layerPrev.neurons! + let indicesPtr = (indices as! MetalSharedBuffer).buffer + + for elem in 0..).buffer + + for elem in 0..).buffer + + if !accumulateDeltaWeights + { + for k in 0..() -> T + { + var losses = [T](repeating: 0.0, count: batchSize) + + if let layerPrev = self.layerPrev as? LayerSeq + { + let neuronsPrev = layerPrev.neurons! + + for elem in 0..() throws -> T + { + try checkLossGPU(batchSize: batchSize) + + let layerPrev = self.layerPrev as! LayerSeq + + let pNbNeurons: [UInt32] = [UInt32(nbNeurons)] + let pNbBatch: [UInt32] = [UInt32(batchSize)] + let pSequence: [UInt32] = [UInt32(sequence)] + + let command = MetalKernel.get.createCommand( + "vqSeqLoss", deviceID: deviceID + ) + command.setBuffer(layerPrev.outs.metal, atIndex: 0) + command.setBuffer(outs.metal, atIndex: 1) + command.setBytes(pNbNeurons, atIndex: 2) + command.setBytes(pNbBatch, atIndex: 3) + command.setBytes(pSequence, atIndex: 4) + command.setBuffer(loss.metal, atIndex: 5) + + command.dispatchThreads(batchSize) + command.enqueue() + + MetalKernel.get.download([loss]) + var loss: Float = 0.0 + let lossPtr = self.loss.buffer + for i in 0.. [IWeightArrays] + { + return [_wArrays] + } + + /// Get the weights in the GPU execution context. + public func collectWeightsGPU() -> [IWeightBuffers] + { + return [_wBuffers] + } +} diff --git a/Sources/GrAIdient/LayerSeq/ValueSeq.swift b/Sources/GrAIdient/LayerSeq/ValueSeq.swift new file mode 100644 index 00000000..9f67df0a --- /dev/null +++ b/Sources/GrAIdient/LayerSeq/ValueSeq.swift @@ -0,0 +1,549 @@ +// +// ValueSeq.swift +// GrAIdient +// +// Created by Jean-François Reboud on 10/03/2023. +// + +/// +/// Layer with a sequential shape neural structure. +/// +/// This layer computes the values (value layer) that are scaled through attention scores (score layer). +/// +public class ValueSeq: LayerMergeSeq +{ + /// Number of heads (groups) of neurons. + let _nbHeads: Int + + private enum Keys: String, CodingKey + { + case nbHeads + } + + /// + /// Create a layer with a sequential shape neural structure. + /// + /// - Parameters: + /// - value: Previous layer containing the value. + /// - score: Previous layer contianing the attention scores per sequence. + /// - nbHeads: Number of heads (groups) of neurons. + /// - params: Contextual parameters linking to the model. + /// + public init(value: LayerSeq, score: LayerSeq, nbHeads: Int, + params: GrAI.Model.Params) throws + { + if value.nbNeurons % nbHeads != 0 + { + throw LayerError.Init(message: + "`nbNeurons` (\(value.nbNeurons)) " + + "should be a multiple of nbHeads (\(nbHeads))." + ) + } + if score.nbNeurons % nbHeads != 0 + { + throw LayerError.Init(message: + "`nbNeurons` (\(score.nbNeurons)) " + + "should be a multiple of nbHeads (\(nbHeads))." + ) + } + if value.sequence != score.sequence + { + throw LayerError.Init(message: "Layer structure error.") + } + + _nbHeads = nbHeads + super.init(layersPrev: [value, score], + sequence: value.sequence, + nbNeurons: value.nbNeurons, + params: params) + } + + /// + /// Decode from the disk. + /// + /// Throw an error if reading from the decoder fails, or + /// if the data read is corrupted or otherwise invalid. + /// + /// - Parameter decoder: The decoder to read data from. + /// + public required init(from decoder: Decoder) throws + { + let values = try decoder.container(keyedBy: Keys.self) + _nbHeads = try values.decode(Int.self, forKey: Keys.nbHeads) + try super.init(from: decoder) + } + + /// + /// Encode to the disk. + /// + /// If the value fails to encode anything, `encoder` will encode an empty + /// keyed container in its place. + /// + /// Throw an error if any values are invalid for the given + /// encoder's format. + /// + /// - Parameter encoder: The encoder to write data to. + /// + public override func encode(to encoder: Encoder) throws + { + var container = encoder.container(keyedBy: Keys.self) + try container.encode(_nbHeads, forKey: Keys.nbHeads) + try super.encode(to: encoder) + } + + /// + /// Create a layer with same values as this. + /// + /// - Parameters: + /// - mapping: Dictionary allowing to find the layer associated to some id. + /// This dictionary is particularly useful when the different layers cannot access + /// their `layerPrev`. + /// - inPlace: Whether hard resources should be copied as is. + /// + /// - Returns: A new layer. When `inPlace` is false, `initKernel` is + /// necessary in order to recreate hard resources. + /// + public override func copy( + mapping: Dictionary, + inPlace: Bool) -> Layer + { + let context = ModelContext(name: "", curID: 0) + let params = GrAI.Model.Params(context: context) + params.context.curID = id + + var layersPrev = [LayerSeq]() + for idPrev in _idsPrev + { + layersPrev.append(mapping[idPrev] as! LayerSeq) + } + + let layer = try! ValueSeq( + value: layersPrev[0], score: layersPrev[1], nbHeads: _nbHeads, + params: params + ) + return layer + } + + /// + /// Apply the forward pass of the Gradient Checking in CPU execution context. + /// + /// Throw an error if batch size is greater than the first batch size. + /// + public override func forwardGCCPU() throws + { + try checkStateCPU(batchSize: batchSize) + + let (nbSameElems, layersIndex, nbElems) = getMergedGraph() + + var nbGC = nbSameElems + for nbElemsTmp in nbElems + { + nbGC += nbElemsTmp + } + + for seq in 0..= 0) + { + outs[id] = 1.0 / (1.0 + exp(-tmps[id])); + } + else + { + outs[id] = exp(tmps[id]) / (1.0 + exp(tmps[id])); + } } kernel void backwardSigmoid( @@ -218,7 +225,89 @@ kernel void backwardSigmoid( return ; } - float tmp = 1.0 / (1.0 + exp(-tmps[id])); + float tmp; + if (tmps[id] >= 0) + { + tmp = 1.0 / (1.0 + exp(-tmps[id])); + } + else + { + tmp = exp(tmps[id]) / (1.0 + exp(tmps[id])); + } + float derivative = tmp * (1 - tmp); delta[id] = delta[id] * derivative; } + +kernel void forwardGELU( + constant uint * pNbElems, + device float * tmps, + device float * outs, + uint id [[ thread_position_in_grid ]]) +{ + uint nbElems; + + if (pNbElems) + { + nbElems = pNbElems[0]; + } + else + return ; + + if (id >= nbElems) + { + return ; + } + + float cst = sqrt(2.0 / 3.14159); + float x = outs[id]; + float tmp1 = cst * (x + 0.044715 * pow(x, 3)); + float tmp2; + if (tmp1 >= 0) + { + tmp2 = (1.0 - exp(-2.0 * tmp1)) / (1.0 + exp(-2.0 * tmp1)); + } + else + { + tmp2 = (exp(2.0 * tmp1) - 1.0) / (exp(2.0 * tmp1) + 1.0); + } + tmps[id] = x; + outs[id] = 0.5 * x * (1 + tmp2); +} + +kernel void backwardGELU( + const device float * tmps, + constant uint * pNbElems, + device float * delta, + uint id [[ thread_position_in_grid ]]) +{ + uint nbElems; + + if (pNbElems) + { + nbElems = pNbElems[0]; + } + else + return ; + + if (id >= nbElems) + { + return ; + } + + float cst = sqrt(2.0 / 3.14159); + float x = tmps[id]; + float tmp1 = cst * (x + 0.044715 * pow(x, 3)); + float tmp2; + if (tmp1 >= 0) + { + tmp2 = (1.0 - exp(-2.0 * tmp1)) / (1.0 + exp(-2.0 * tmp1)); + } + else + { + tmp2 = (exp(2.0 * tmp1) - 1.0) / (exp(2.0 * tmp1) + 1.0); + } + float tmp3 = cst * (1 + 3 * 0.044715 * x * x) * (1 - tmp2 * tmp2); + float derivative = 0.5 * (1 + tmp2 + x * tmp3); + delta[id] = delta[id] * derivative; +} diff --git a/Sources/GrAIdient/Metal/Kernel/BN.metal b/Sources/GrAIdient/Metal/Kernel/BatchNorm.metal similarity index 95% rename from Sources/GrAIdient/Metal/Kernel/BN.metal rename to Sources/GrAIdient/Metal/Kernel/BatchNorm.metal index 1dae7fa3..413ab070 100644 --- a/Sources/GrAIdient/Metal/Kernel/BN.metal +++ b/Sources/GrAIdient/Metal/Kernel/BatchNorm.metal @@ -1,5 +1,5 @@ // -// BN.metal +// BatchNorm.metal // GrAIdient // // Created by Jean-François Reboud on 14/10/2022. @@ -8,7 +8,7 @@ #include using namespace metal; -kernel void computeConvμ( +kernel void computeBNConvμ( const device float * tmps, constant uint * pNbChannels, constant uint * pNbBatch, @@ -67,7 +67,7 @@ kernel void computeConvμ( } } -kernel void computeConvσ2( +kernel void computeBNConvσ2( const device float * tmps, const device float * μ, constant uint * pNbChannels, @@ -285,9 +285,9 @@ kernel void backwardWeightsBNConv( float deltaTmp = delta[offset]; float xHatTmp = xHat[offset]; - float dxhat = Ɣ[depth] * deltaTmp; - tmp1 += dxhat; - tmp2 += dxhat * xHatTmp; + float dxHat = Ɣ[depth] * deltaTmp; + tmp1 += dxHat; + tmp2 += dxHat * xHatTmp; tmp3 += deltaTmp * xHatTmp; tmp4 += deltaTmp; @@ -326,8 +326,8 @@ kernel void backwardBNConvTraining( uint height; float Ɛ = 1e-5; - if (pNbChannels && pNbBatch && pDimensions && σ2 && xHat && Ɣ && - delta) + if (pNbChannels && pNbBatch && pDimensions && + σ2 && xHat && Ɣ && sum1 && sum2 && delta) { nbChannels = *pNbChannels; nbBatch = *pNbBatch; @@ -353,8 +353,8 @@ kernel void backwardBNConvTraining( uint offset = j + (offsetStart + i) * width; float mult = 1.0 / ((float)nbElems * sqrt(σ2[depth] + Ɛ)); - float dxhat = Ɣ[depth] * delta[offset]; - float tmp1 = nbElems * dxhat; + float dxHat = Ɣ[depth] * delta[offset]; + float tmp1 = nbElems * dxHat; float tmp2 = sum1[depth]; float tmp3 = xHat[offset] * sum2[depth]; @@ -378,8 +378,7 @@ kernel void backwardBNConvInference( uint height; float Ɛ = 1e-5; - if (pNbChannels && pNbBatch && pM && pDimensions && Ɣ && - Eσ2 && delta) + if (pNbChannels && pNbBatch && pM && pDimensions && Ɣ && Eσ2 && delta) { nbChannels = *pNbChannels; nbBatch = *pNbBatch; diff --git a/Sources/GrAIdient/Metal/Kernel/Convolution.metal b/Sources/GrAIdient/Metal/Kernel/Convolution.metal index 28f65073..220e4c0b 100644 --- a/Sources/GrAIdient/Metal/Kernel/Convolution.metal +++ b/Sources/GrAIdient/Metal/Kernel/Convolution.metal @@ -30,6 +30,7 @@ kernel void convForward( uint nbChannelsPrev; int startI, startJ; int endI, endJ; + int offI, offJ; uint stride; uint nbBatch; @@ -50,6 +51,8 @@ kernel void convForward( endI = pStart[1]; startJ = pStart[2]; endJ = pStart[3]; + offI = pStart[4]; + offJ = pStart[5]; stride = pStride[0]; } else @@ -79,11 +82,13 @@ kernel void convForward( for (int k=startI; k<=endI; k++){ for (int l=startJ; l<=endJ; l++) { - if ((int)(stride*j)+l >= 0 && stride*j+l < widthPrev - && (int)(stride*i)+k >= 0 && stride*i+k < heightPrev) + if ((int)(stride*j)+l-offJ >= 0 && + (int)(stride*j)+l-offJ < (int)widthPrev && + (int)(stride*i)+k-offI >= 0 && + (int)(stride*i)+k-offI < (int)heightPrev) { - uint offsetPrev = stride*j+l + - (offsetStartPrev + stride*i+k)*widthPrev; + uint offsetPrev = (int)(stride*j)+l-offJ + + (offsetStartPrev + (int)(stride*i)+k-offI)*widthPrev; float outPrev = outsPrev[offsetPrev]; uint offsetWeights = l-startJ + @@ -121,6 +126,7 @@ kernel void convBackward( uint nbChannelsPrev; int startI, startJ; int endI, endJ; + int offI, offJ; uint stride; uint nbBatch; uint dirty; @@ -142,6 +148,8 @@ kernel void convBackward( endI = pStart[1]; startJ = pStart[2]; endJ = pStart[3]; + offI = pStart[4]; + offJ = pStart[5]; stride = pStride[0]; dirty = *pDirty; } @@ -171,10 +179,10 @@ kernel void convBackward( for (int k=startI; k<=endI; k++){ for (int l=startJ; l<=endJ; l++) { - if ((i-k) % stride == 0 && (j-l) % stride == 0) + if ((i-k+offI) % stride == 0 && (j-l+offJ) % stride == 0) { - int i1 = (i-k) / stride; - int j1 = (j-l) / stride; + int i1 = (i-k+offI) / stride; + int j1 = (j-l+offJ) / stride; if (j1 >= 0 && j1 < (int)width && i1 >= 0 && i1 < (int)height) @@ -225,6 +233,7 @@ kernel void convBatchDerWeights( uint nbChannelsPrev; int startI, startJ; int endI, endJ; + int offI, offJ; uint stride; uint nbBatch; uint accumulate; @@ -246,6 +255,8 @@ kernel void convBatchDerWeights( endI = pStart[1]; startJ = pStart[2]; endJ = pStart[3]; + offI = pStart[4]; + offJ = pStart[5]; stride = pStride[0]; accumulate = *pAccumulate; } @@ -278,14 +289,16 @@ kernel void convBatchDerWeights( for (uint k=0; k= 0 && stride*l+j < widthPrev && - (int)(stride*k)+i >= 0 && stride*k+i < heightPrev) + if ((int)(stride*l)+j-offJ >= 0 && + (int)(stride*l)+j-offJ < (int)widthPrev && + (int)(stride*k)+i-offI >= 0 && + (int)(stride*k)+i-offI < (int)heightPrev) { uint offset = l + (offsetStart + k) * width; float deltaCur = delta[offset]; - uint offsetPrev = stride*l+j + - (offsetStartPrev + stride*k+i)*widthPrev; + uint offsetPrev = (int)(stride*l)+j-offJ + + (offsetStartPrev + (int)(stride*k)+i-offI)*widthPrev; float outPrev = outsPrev[offsetPrev]; tmp += deltaCur * outPrev; @@ -384,6 +397,7 @@ kernel void convDerWeights( uint nbChannelsPrev; int startI, startJ; int endI, endJ; + int offI, offJ; uint stride; uint nbBatch; @@ -404,6 +418,8 @@ kernel void convDerWeights( endI = pStart[1]; startJ = pStart[2]; endJ = pStart[3]; + offI = pStart[4]; + offJ = pStart[5]; stride = pStride[0]; } else @@ -439,14 +455,16 @@ kernel void convDerWeights( for (uint k=0; k= 0 && stride*l+j < widthPrev && - (int)(stride*k)+i >= 0 && stride*k+i < heightPrev) + if ((int)(stride*l)+j-offJ >= 0 && + (int)(stride*l)+j-offJ < (int)widthPrev && + (int)(stride*k)+i-offI >= 0 && + (int)(stride*k)+i-offI < (int)heightPrev) { uint offset = l + (offsetStart + k) * width; float deltaCur = delta[offset]; - uint offsetPrev = stride*l+j + - (offsetStartPrev + stride*k+i)*widthPrev; + uint offsetPrev = (int)(stride*l)+j-offJ + + (offsetStartPrev + (int)(stride*k)+i-offI)*widthPrev; float outPrev = outsPrev[offsetPrev]; tmp += deltaCur * outPrev; diff --git a/Sources/GrAIdient/Metal/Kernel/Deconvolution.metal b/Sources/GrAIdient/Metal/Kernel/Deconvolution.metal index 18cc7f38..28308ee0 100644 --- a/Sources/GrAIdient/Metal/Kernel/Deconvolution.metal +++ b/Sources/GrAIdient/Metal/Kernel/Deconvolution.metal @@ -79,10 +79,10 @@ kernel void deconvForward( for (int k=startI; k<=endI; k++){ for (int l=startJ; l<=endJ; l++) { - if ((i+k-endI) % stride == 0 && (j+l-endJ) % stride == 0) + if ((i-k+startI) % stride == 0 && (j-l+startJ) % stride == 0) { - int i1 = (i+k-endI) / stride; - int j1 = (j+l-endJ) / stride; + int i1 = (i-k+startI) / stride; + int j1 = (j-l+startJ) / stride; if (j1 >= 0 && j1 < (int)widthPrev && i1 >= 0 && i1 < (int)heightPrev) @@ -177,11 +177,13 @@ kernel void deconvBackward( for (int k=startI; k<=endI; k++){ for (int l=startJ; l<=endJ; l++) { - if ((int)(stride*j)+endJ-l >= 0 && stride*j+endJ-l < width - && (int)(stride*i)+endI-k >= 0 && stride*i+endI-k < height) + if ((int)(stride*j)+l-startJ >= 0 && + (int)(stride*j)+l-startJ < (int)width && + (int)(stride*i)+k-startI >= 0 && + (int)(stride*i)+k-startI < (int)height) { - uint offset = stride*j+endJ-l + - (offsetStart + stride*i+endI-k) * width; + uint offset = (int)(stride*j)+l-startJ + + (offsetStart + (int)(stride*i)+k-startI) * width; float deltaCur = delta[offset]; uint offsetWeights = l-startJ + @@ -279,10 +281,10 @@ kernel void deconvBatchDerWeights( for (uint k=0; k= 0 && j1 < (int)widthPrev && i1 >= 0 && i1 < (int)heightPrev) @@ -391,10 +393,10 @@ kernel void deconvDerWeights( for (uint k=0; k= 0 && j1 < (int)widthPrev && i1 >= 0 && i1 < (int)heightPrev) diff --git a/Sources/GrAIdient/Metal/Kernel/FullyConnectedPatch.metal b/Sources/GrAIdient/Metal/Kernel/FullyConnectedPatch.metal new file mode 100644 index 00000000..9b5ee8e1 --- /dev/null +++ b/Sources/GrAIdient/Metal/Kernel/FullyConnectedPatch.metal @@ -0,0 +1,480 @@ +// +// FullyConnectedPatch.metal +// GrAIdient +// +// Created by Jean-François Reboud on 25/02/2023. +// + +#include +using namespace metal; + +kernel void flPatchForward( + const device float * outsPrev, + const device float * weights, + const device float * biases, + constant uint * pNbNeurons, + constant uint * pNbChannelsPrev, + constant uint * pDimensionsPrev, + constant uint * pPatch, + constant uint * pNbBatch, + constant uint * pSequence, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbChannelsPrev; + uint heightPrev, widthPrev; + uint patch; + uint nbBatch; + uint sequence; + + if (pNbNeurons && pNbChannelsPrev && pDimensionsPrev && pPatch && + pNbBatch && pSequence && + outsPrev && weights && biases && outs) + { + nbNeurons = *pNbNeurons; + nbChannelsPrev = *pNbChannelsPrev; + widthPrev = pDimensionsPrev[0]; + heightPrev = pDimensionsPrev[1]; + patch = *pPatch; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint weightWidth = nbChannelsPrev * patch * patch; + uint depth = id[0]; + uint elem = id[1] / sequence; + uint seq = id[1] % sequence; + + if (depth >= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint nbSeqPerCol = widthPrev / patch; + uint seqI = seq / nbSeqPerCol; + uint seqJ = seq % nbSeqPerCol; + + uint iStart = seqI * patch; + uint jStart = seqJ * patch; + + float tmp = biases[depth]; + for (uint depthPrev=0; depthPrev= weightWidth || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint nbSeqPerCol = widthPrev / patch; + uint seqI = seq / nbSeqPerCol; + uint seqJ = seq % nbSeqPerCol; + + uint iStart = seqI * patch; + uint jStart = seqJ * patch; + + uint res = offsetWeight; + uint depthPrev = res / (patch * patch); + res -= depthPrev * patch * patch; + uint i = res / patch; + res -= i * patch; + uint j = res; + + float tmp = 0.0; + for (uint depth=0; depth= nbNeurons || offsetWeight >= weightWidth) + { + return ; + } + + uint nbSeqPerCol = widthPrev / patch; + + uint res = offsetWeight; + uint depthPrev = res / (patch * patch); + res -= depthPrev * patch * patch; + uint i = res / patch; + res -= i * patch; + uint j = res; + + float tmp = 0.0; + for (uint elem=0; elem= nbNeurons) + { + return ; + } + + float tmp = 0.0; + for (uint elem=0; elem= nbNeurons * nbBatch || + offsetWeight >= weightWidth) + { + return ; + } + + uint nbSeqPerCol = widthPrev / patch; + + uint res = offsetWeight; + uint depthPrev = res / (patch * patch); + res -= depthPrev * patch * patch; + uint i = res / patch; + res -= i * patch; + uint j = res; + + float tmp = 0.0; + for (uint seq=0; seq= nbNeurons || elem >= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint seq=0; seq= nbNeurons || offsetWeight >= weightWidth) + { + return ; + } + + uint offsetWeights = offsetWeight + weightWidth * depth; + + float tmp = 0.0; + for (uint elem=0; elem +using namespace metal; + +kernel void flSeqForward( + const device float * outsPrev, + const device float * weights, + const device float * biases, + constant uint * pNbNeurons, + constant uint * pNbNeuronsPrev, + constant uint * pNbBatch, + constant uint * pSequence, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbNeuronsPrev; + uint nbBatch; + uint sequence; + + if (pNbNeurons && pNbNeuronsPrev && pNbBatch && pSequence && + outsPrev && weights && biases && outs) + { + nbNeurons = *pNbNeurons; + nbNeuronsPrev = *pNbNeuronsPrev; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1] / sequence; + uint seq = id[1] % sequence; + + if (depth >= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + float tmp = biases[depth]; + for (uint depthPrev=0; depthPrev= nbNeuronsPrev || elem >= nbBatch || seq >= sequence) + { + return ; + } + + float tmp = 0.0; + for (uint depth=0; depth= nbNeurons || depthPrev >= nbNeuronsPrev) + { + return ; + } + + float tmp = 0.0; + for (uint elem=0; elem= nbNeurons * nbBatch || + depthPrev >= nbNeuronsPrev) + { + return ; + } + + float tmp = 0.0; + for (uint seq=0; seq= nbNeurons || depthPrev >= nbNeuronsPrev) + { + return ; + } + + uint offsetWeights = depthPrev + nbNeuronsPrev * depth; + + float tmp = 0.0; + for (uint elem=0; elem +using namespace metal; + +kernel void computeInstanceNormConvμ( + const device float * tmps, + constant uint * pNbChannels, + constant uint * pNbBatch, + constant uint * pDimensions, + device float * μ, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbChannels; + uint nbBatch; + uint width; + uint height; + + if (pNbChannels && pNbBatch && pDimensions && tmps && μ) + { + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + width = pDimensions[0]; + height = pDimensions[1]; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1]; + if (depth >= nbChannels || elem >= nbBatch) + { + return ; + } + + uint nbElems = width * height; + float sum = 0.0; + + for (uint x=0; x= nbChannels || elem >= nbBatch) + { + return ; + } + + uint nbElems = width * height; + float sum = 0.0; + + for (uint x=0; x= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float tmp1 = tmps[offset] - μ[depth + nbChannels * elem]; + float tmp2 = sqrt(σ2[depth + nbChannels * elem] + Ɛ); + float xhat = tmp1 / tmp2; + xHat[offset] = xhat; + tmps[offset] = Ɣ[depth] * xhat + β[depth]; +} + +kernel void forwardAdaIN( + const device float * outsPrev, + const device float * styles, + const device float * μ, + const device float * σ2, + constant uint * pNbChannels, + constant uint * pNbBatch, + constant uint * pDimensions, + device float * outs, + device float * xHat, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbChannels; + uint nbBatch; + uint width; + uint height; + float Ɛ = 1e-5; + + if (pNbChannels && pNbBatch && pDimensions && outsPrev && styles && + outs && xHat && μ && σ2) + { + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + width = pDimensions[0]; + height = pDimensions[1]; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float tmp1 = outsPrev[offset] - μ[depth + nbChannels * elem]; + float tmp2 = sqrt(σ2[depth + nbChannels * elem] + Ɛ); + float xhat = tmp1 / tmp2; + xHat[offset] = xhat; + outs[offset] = styles[depth] * xhat + styles[depth + nbChannels]; +} + +kernel void backwardWeightsInstanceNormConv( + const device float * delta, + const device float * xHat, + const device float * Ɣ, + constant uint * pNbChannels, + constant uint * pNbBatch, + constant uint * pDimensions, + constant uint * pAccumulate, + device float * sum1, + device float * sum2, + device float * dƔ, + device float * dβ, + uint id [[ thread_position_in_grid ]]) +{ + uint nbChannels; + uint nbBatch; + uint width; + uint height; + uint accumulate; + + if (pNbChannels && pNbBatch && pDimensions && pAccumulate && + delta && xHat && Ɣ && + sum1 && sum2 && dƔ && dβ) + { + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + width = pDimensions[0]; + height = pDimensions[1]; + accumulate = *pAccumulate; + } + else + return ; + + uint depth = id; + if (depth >= nbChannels) + { + return ; + } + + float tmp3 = 0.0, tmp4 = 0.0; + for (uint elem=0; elem= nbChannels || elem >= nbBatch) + { + return ; + } + + float tmp1 = 0.0, tmp2 = 0.0; + float tmp3 = 0.0, tmp4 = 0.0; + + for (uint x=0; x= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float mult = + 1.0 / ((float)nbElems * sqrt(σ2[depth + nbChannels * elem] + Ɛ)); + float dxHat = Ɣ[depth] * delta[offset]; + float tmp1 = nbElems * dxHat; + float tmp2 = sum1[depth + nbChannels * elem]; + float tmp3 = xHat[offset] * sum2[depth + nbChannels * elem]; + + delta[offset] = mult * (tmp1 - tmp2 - tmp3); +} + +kernel void backward1AdaIN( + const device float * delta, + const device float * σ2, + const device float * xHat, + const device float * styles, + const device float * sum1, + const device float * sum2, + constant uint * pNbChannels, + constant uint * pNbBatch, + constant uint * pDimensions, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbChannels; + uint nbBatch; + uint width; + uint height; + uint dirty; + float Ɛ = 1e-5; + + if (pNbChannels && pNbBatch && pDimensions && pDirty && + delta && σ2 && xHat && styles && sum1 && sum2 && deltaPrev) + { + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + width = pDimensions[0]; + height = pDimensions[1]; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + uint nbElems = width * height; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float mult = + 1.0 / ((float)nbElems * sqrt(σ2[depth + nbChannels * elem] + Ɛ)); + float dxHat = styles[depth] * delta[offset]; + float tmp1 = nbElems * dxHat; + float tmp2 = sum1[depth + nbChannels * elem]; + float tmp3 = xHat[offset] * sum2[depth + nbChannels * elem]; + + if (dirty) + { + deltaPrev[offset] = mult * (tmp1 - tmp2 - tmp3); + } + else + { + deltaPrev[offset] += mult * (tmp1 - tmp2 - tmp3); + } +} diff --git a/Sources/GrAIdient/Metal/Kernel/Layer1D.metal b/Sources/GrAIdient/Metal/Kernel/Layer1D.metal index f5f1f3c6..be768aee 100644 --- a/Sources/GrAIdient/Metal/Kernel/Layer1D.metal +++ b/Sources/GrAIdient/Metal/Kernel/Layer1D.metal @@ -54,18 +54,22 @@ kernel void MSE1DLossDerivative( constant uint * pNbNeurons, constant float * pCoeff, constant uint * pNbBatch, + constant uint * pDirty, device float * deltaPrev, uint2 id [[ thread_position_in_grid ]]) { uint nbNeurons; float coeff; uint nbBatch; + uint dirty; - if (pNbNeurons && pNbBatch && pCoeff && outs && groundTruth && deltaPrev) + if (pNbNeurons && pNbBatch && pCoeff && pDirty && + outs && groundTruth && deltaPrev) { nbNeurons = *pNbNeurons; coeff = *pCoeff; nbBatch = *pNbBatch; + dirty = *pDirty; } else return ; @@ -84,7 +88,14 @@ kernel void MSE1DLossDerivative( float out = outs[offset]; float diff = out - gt; - deltaPrev[offset] = 2 * coeff * diff / float(nbNeurons * nbBatch); + if (dirty) + { + deltaPrev[offset] = 2 * coeff * diff / float(nbNeurons * nbBatch); + } + else + { + deltaPrev[offset] += 2 * coeff * diff / float(nbNeurons * nbBatch); + } } kernel void linearErrorLoss( @@ -132,18 +143,21 @@ kernel void linearErrorLossDerivative( constant uint * pNbNeurons, constant float * pCoeff, constant uint * pNbBatch, + constant uint * pDirty, device float * deltaPrev, uint2 id [[ thread_position_in_grid ]]) { uint nbNeurons; float coeff; uint nbBatch; + uint dirty; - if (pNbNeurons && pNbBatch && pCoeff && outs && deltaPrev) + if (pNbNeurons && pNbBatch && pCoeff && pDirty && outs && deltaPrev) { nbNeurons = *pNbNeurons; coeff = *pCoeff; nbBatch = *pNbBatch; + dirty = *pDirty; } else return ; @@ -157,7 +171,15 @@ kernel void linearErrorLossDerivative( } uint offset = depth + nbNeurons * elem; - deltaPrev[offset] = coeff / float(nbNeurons * nbBatch); + + if (dirty) + { + deltaPrev[offset] = coeff / float(nbNeurons * nbBatch); + } + else + { + deltaPrev[offset] += coeff / float(nbNeurons * nbBatch); + } } kernel void selectNeurons1DForward( @@ -324,112 +346,118 @@ kernel void concat1DBackward( kernel void softmax1DForward( const device float * outsPrev, - constant uint * pSize, + constant uint * pNbHeads, constant uint * pNbNeurons, constant uint * pNbBatch, device float * outs, uint2 id [[ thread_position_in_grid ]]) { + uint nbHeads; uint size; uint nbNeurons; uint nbBatch; - if (pSize && pNbNeurons && pNbBatch && outsPrev && outs) + if (pNbHeads && pNbNeurons && pNbBatch && outsPrev && outs) { - size = *pSize; + nbHeads = *pNbHeads; nbNeurons = *pNbNeurons; nbBatch = *pNbBatch; + size = nbNeurons / nbHeads; } else return ; uint depth = id[0]; uint elem = id[1]; - uint block = depth / size; + uint head = depth / size; if (depth >= nbNeurons || elem >= nbBatch) { return ; } + float cMax = outsPrev[0+head*size + nbNeurons * elem]; + for (uint j=0; j cMax) + { + cMax = outPrev; + } + } + float sum1 = 0.0; - for (uint j1=0; j1= nbNeurons || elem >= nbBatch) { return ; } - float sum1 = 0.0; - for (uint j1=0; j1= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint depth=0; depth= nbNeurons || elem >= nbBatch) + { + return ; + } + + uint offset = depth + nbNeurons * elem; + + float gt = groundTruth[offset]; + float out = outs[offset]; + float derivative = 0.0; + + if (gt == 1.0) + { + derivative = -1 / out; + } + else if (gt == 0.0) + { + derivative = 1 / (1 - out); + } + + if (dirty) + { + deltaPrev[offset] = coeff * derivative / float(nbNeurons * nbBatch); + } + else + { + deltaPrev[offset] += coeff * derivative / float(nbNeurons * nbBatch); + } +} + +kernel void BCESigmoid1DLoss( + const device float * outs, + const device float * groundTruth, + constant uint * pNbNeurons, + constant uint * pNbBatch, + device float * losses, + uint id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + + if (pNbNeurons && pNbBatch && outs && groundTruth && losses) + { + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + } + else + return ; + + uint elem = id; + if (elem >= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint depth=0; depth 0) + { + value = (1 - gt) * out; + value += log(1 + exp(-out)); + } + else + { + value = -out * gt; + value += log(exp(out) + 1); + } + + tmp += value; + } + + losses[elem] = tmp; +} + +kernel void BCESigmoid1DLossDerivative( + const device float * outs, + const device float * groundTruth, + constant uint * pNbNeurons, + constant float * pCoeff, + constant uint * pNbBatch, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + float coeff; + uint nbBatch; + uint dirty; + + if (pNbNeurons && pNbBatch && pCoeff && pDirty && + outs && groundTruth && deltaPrev) + { + nbNeurons = *pNbNeurons; + coeff = *pCoeff; + nbBatch = *pNbBatch; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1]; + + if (depth >= nbNeurons || elem >= nbBatch) + { + return ; + } + + uint offset = depth + nbNeurons * elem; + + float gt = groundTruth[offset]; + float out = outs[offset]; + float value; + + if (out >= 0) + { + value = 1.0 / (1.0 + exp(-out)); + } + else + { + value = exp(out) / (1.0 + exp(out)); + } + + if (dirty) + { + deltaPrev[offset] = coeff * (value - gt) / float(nbNeurons * nbBatch); + } + else + { + deltaPrev[offset] += coeff * (value - gt) / float(nbNeurons * nbBatch); + } +} diff --git a/Sources/GrAIdient/Metal/Kernel/Layer2D.metal b/Sources/GrAIdient/Metal/Kernel/Layer2D.metal index 0a738313..32d8dccb 100644 --- a/Sources/GrAIdient/Metal/Kernel/Layer2D.metal +++ b/Sources/GrAIdient/Metal/Kernel/Layer2D.metal @@ -162,11 +162,13 @@ kernel void maxPoolForward( for (int k=start; k<=end; k++){ for (int l=start; l<=end; l++) { - if ((int)(stride*j)+l >= 0 && stride*j+l < widthPrev - && (int)(stride*i)+k >= 0 && stride*i+k < heightPrev) + if ((int)(stride*j)+l >= 0 && + (int)(stride*j)+l < (int)widthPrev && + (int)(stride*i)+k >= 0 && + (int)(stride*i)+k < (int)heightPrev) { - uint offsetPrev = stride*j+l + - (offsetStartPrev + stride*i+k)*widthPrev; + uint offsetPrev = (int)(stride*j)+l + + (offsetStartPrev + (int)(stride*i)+k)*widthPrev; float outPrev = outsPrev[offsetPrev]; if (outPrev > maxVal) @@ -1316,6 +1318,7 @@ kernel void resizeBilinearPadForward( constant uint * pDimensions, constant uint * pDimensionsPrev, constant uint * pDimensionsResize, + constant uint * pPadDimensions, constant float * pPadValue, constant uint * pNbBatch, device float * outs, @@ -1325,11 +1328,13 @@ kernel void resizeBilinearPadForward( uint heightPrev, widthPrev; uint heightResize, widthResize; uint nbChannels; + uint padStartI, padEndI; + uint padStartJ, padEndJ; float padValue; uint nbBatch; if (pNbChannels && pDimensions && pDimensionsPrev && pDimensionsResize && - pPadValue && pNbBatch && outsPrev && outs) + pPadDimensions && pPadValue && pNbBatch && outsPrev && outs) { width = pDimensions[0]; height = pDimensions[1]; @@ -1337,6 +1342,10 @@ kernel void resizeBilinearPadForward( heightPrev = pDimensionsPrev[1]; widthResize = pDimensionsResize[0]; heightResize = pDimensionsResize[1]; + padStartI = pPadDimensions[0]; + padEndI = pPadDimensions[1]; + padStartJ = pPadDimensions[2]; + padEndJ = pPadDimensions[3]; padValue = *pPadValue; nbChannels = *pNbChannels; nbBatch = *pNbBatch; @@ -1357,21 +1366,19 @@ kernel void resizeBilinearPadForward( float ratioInOutI = float(heightPrev - 1) / float(heightResize - 1); float ratioInOutJ = float(widthPrev - 1) / float(widthResize - 1); - float padDimensionI = (height - heightResize) / 2; - float padDimensionJ = (width - widthResize) / 2; uint offsetStart = (depth + nbChannels * elem) * height; uint offset = j + (offsetStart + i) * width; - if (i < padDimensionI || i >= height - padDimensionI || - j < padDimensionJ || j >= width - padDimensionJ) + if (i < padStartI || i >= height - padEndI || + j < padStartJ || j >= width - padEndJ) { outs[offset] = padValue; } else { - float I = i-padDimensionI; - float J = j-padDimensionJ; + float I = i-padStartI; + float J = j-padStartJ; float iPrev = I * ratioInOutI; float jPrev = J * ratioInOutJ; @@ -1405,6 +1412,7 @@ kernel void resizeBilinearPadBackward( constant uint * pDimensions, constant uint * pDimensionsPrev, constant uint * pDimensionsResize, + constant uint * pPadDimensions, constant uint * pNbBatch, device float * deltaPrev, uint2 id [[ thread_position_in_grid ]]) @@ -1413,10 +1421,12 @@ kernel void resizeBilinearPadBackward( uint heightPrev, widthPrev; uint heightResize, widthResize; uint nbChannels; + uint padStartI, padEndI; + uint padStartJ, padEndJ; uint nbBatch; if (pNbChannels && pDimensions && pDimensionsPrev && pDimensionsResize && - pNbBatch && delta && deltaPrev) + pPadDimensions && pNbBatch && delta && deltaPrev) { width = pDimensions[0]; height = pDimensions[1]; @@ -1424,6 +1434,10 @@ kernel void resizeBilinearPadBackward( heightPrev = pDimensionsPrev[1]; widthResize = pDimensionsResize[0]; heightResize = pDimensionsResize[1]; + padStartI = pPadDimensions[0]; + padEndI = pPadDimensions[1]; + padStartJ = pPadDimensions[2]; + padEndJ = pPadDimensions[3]; nbChannels = *pNbChannels; nbBatch = *pNbBatch; } @@ -1443,8 +1457,6 @@ kernel void resizeBilinearPadBackward( float ratioInOutI = float(heightPrev - 1) / float(heightResize - 1); float ratioInOutJ = float(widthPrev - 1) / float(widthResize - 1); - float padDimensionI = (height - heightResize) / 2; - float padDimensionJ = (width - widthResize) / 2; uint offsetStart = (depth + nbChannels * elem) * height; uint offsetStartPrev = (depth + nbChannels * elem) * heightPrev; @@ -1479,8 +1491,8 @@ kernel void resizeBilinearPadBackward( if (kPrevInf == i && lPrevInf == j) { - uint offset = l+padDimensionJ + - (offsetStart + k+padDimensionI) * width; + uint offset = l+padStartJ + + (offsetStart + k+padStartI) * width; float deltaCur = delta[offset]; deltaPrev[offsetPrev] += @@ -1488,24 +1500,24 @@ kernel void resizeBilinearPadBackward( } else if (kPrevInf == i && lPrevSup == j) { - uint offset = l+padDimensionJ + - (offsetStart + k+padDimensionI) * width; + uint offset = l+padStartJ + + (offsetStart + k+padStartI) * width; float deltaCur = delta[offset]; deltaPrev[offsetPrev] += deltaCur * (1.0 - kWeight) * lWeight; } else if (kPrevSup == i && lPrevInf == j) { - uint offset = l+padDimensionJ + - (offsetStart + k+padDimensionI) * width; + uint offset = l+padStartJ + + (offsetStart + k+padStartI) * width; float deltaCur = delta[offset]; deltaPrev[offsetPrev] += deltaCur * kWeight * (1.0 - lWeight); } else if (kPrevSup == i && lPrevSup == j) { - uint offset = l+padDimensionJ + - (offsetStart + k+padDimensionI) * width; + uint offset = l+padStartJ + + (offsetStart + k+padStartI) * width; float deltaCur = delta[offset]; deltaPrev[offsetPrev] += deltaCur * kWeight * lWeight; @@ -1849,7 +1861,109 @@ kernel void resizeBilinearCropBackward( }} } -kernel void concat2DForward( +kernel void concat02DForward( + const device float * outsPrev, + constant uint * pGlobalOffset, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbBatch; + uint globalOffset; + + if (pGlobalOffset && pNbChannels && pDimensions && + pNbBatch && outsPrev && outs) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + globalOffset = *pGlobalOffset; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStartPrev = (depth + nbChannels * elem) * height; + uint offsetStart = (depth + nbChannels * (globalOffset+elem)) * height; + + uint offsetPrev = j + (offsetStartPrev + i) * width; + uint offset = j + (offsetStart + i) * width; + + outs[offset] = outsPrev[offsetPrev]; +} + +kernel void concat02DBackward( + const device float * delta, + constant uint * pGlobalOffset, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbBatch; + uint globalOffset; + uint dirty; + + if (pGlobalOffset && pNbChannels && pDimensions && + pNbBatch && pDirty && delta && deltaPrev) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + globalOffset = *pGlobalOffset; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStartPrev = (depth + nbChannels * elem) * height; + uint offsetStart = (depth + nbChannels * (globalOffset+elem)) * height; + + uint offsetPrev = j + (offsetStartPrev + i) * width; + uint offset = j + (offsetStart + i) * width; + + if (dirty) + { + deltaPrev[offsetPrev] = delta[offset]; + } + else + { + deltaPrev[offsetPrev] += delta[offset]; + } +} + +kernel void concat12DForward( const device float * outsPrev, constant uint * pGlobalOffset, constant uint * pNbChannels, @@ -1898,7 +2012,7 @@ kernel void concat2DForward( outs[offset] = outsPrev[offsetPrev]; } -kernel void concat2DBackward( +kernel void concat12DBackward( const device float * delta, constant uint * pGlobalOffset, constant uint * pNbChannels, @@ -1956,3 +2070,1442 @@ kernel void concat2DBackward( deltaPrev[offsetPrev] += delta[offset]; } } + +kernel void constant2DForward( + const device float * weights, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbBatch; + + if (pNbChannels && pDimensions && pNbBatch && weights && outs) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + outs[offset] = weights[depth]; +} + +kernel void MSE2DLoss( + const device float * outs, + const device float * groundTruth, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + device float * losses, + uint id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbBatch; + + if (pNbChannels && pDimensions && pNbBatch && outs && groundTruth && losses) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + } + else + return ; + + uint elem = id; + if (elem >= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint depth=0; depth= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float gt = groundTruth[offset]; + float out = outs[offset]; + float diff = out - gt; + + if (dirty) + { + deltaPrev[offset] = 2 * coeff * diff / + float(nbBatch * nbChannels * height * width); + } + else + { + deltaPrev[offset] += 2 * coeff * diff / + float(nbBatch * nbChannels * height * width); + } +} + +kernel void selfCorrelate2DForward( + const device float * outsPrev, + constant uint * pNbChannelsPrev, + constant uint * pDimensionsPrev, + constant uint * pNbBatch, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint heightPrev, widthPrev; + uint nbChannelsPrev; + uint nbBatch; + + if (pNbChannelsPrev && pDimensionsPrev && pNbBatch && + outsPrev && outs) + { + widthPrev = pDimensionsPrev[0]; + heightPrev = pDimensionsPrev[1]; + nbChannelsPrev = *pNbChannelsPrev; + nbBatch = *pNbBatch; + } + else + return ; + + uint channel1 = id[0] / nbChannelsPrev; + uint channel2 = id[0] % nbChannelsPrev; + uint elem = id[1]; + + if (channel1 * channel2 >= nbChannelsPrev * nbChannelsPrev || + elem >= nbBatch) + { + return ; + } + + uint offsetStart1 = (channel1 + nbChannelsPrev * elem) * heightPrev; + uint offsetStart2 = (channel2 + nbChannelsPrev * elem) * heightPrev; + + float correlation = 0.0; + for (uint i=0; i= heightPrev * nbBatch || + j * depthPrev >= widthPrev * nbChannelsPrev) + { + return ; + } + + float correlation = 0.0; + for (uint col=0; col= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + float norm = 0.0; + for (uint depth1=0; depth1= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + float normTmp = 0.0; + for (uint depth1=0; depth1 1e-12) + { + for (uint depth1=0; depth1= nbChannels * height * width || + elem >= nbBatch) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float outPrev = outsPrev[offset]; + normShared[threadId[0]] = outPrev * outPrev; + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint stride=threadsPerThreadgroup/2; stride>0; stride>>=1) + { + uint index = threadId[0] + groupId[0] * threadsPerThreadgroup; + if (threadId[0] < stride && + (index + stride) < nbChannels * height * width) + { + normShared[threadId[0]] += normShared[threadId[0] + stride]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + if (threadId[0] == 0) + { + uint offset = elem * nbThreadgroups + groupId[0]; + squaredNorms[offset] = normShared[0]; + } +} + +kernel void normalize122DForward( + const device float * outsPrev, + const device float * squaredNorms, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbThreadgroups, + constant uint * pNbBatch, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbThreadgroups; + uint nbBatch; + + if (pNbChannels && pDimensions && pNbThreadgroups && pNbBatch && + outsPrev && squaredNorms && outs) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbThreadgroups = *pNbThreadgroups; + nbBatch = *pNbBatch; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float norm = sqrt(squaredNorms[elem]); + float outPrev = outsPrev[offset]; + + outs[offset] = outPrev / max(norm, 1e-12); +} + +kernel void computeDeltaTmp122D( + const device float * delta, + const device float * outsPrev, + const device float * squaredNorms, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbThreadgroups, + constant uint * pNbBatch, + device float * deltaTmp, + uint2 groupId [[ threadgroup_position_in_grid ]], + uint2 threadId [[ thread_position_in_threadgroup ]], + uint2 id [[ thread_position_in_grid ]]) +{ + constexpr uint threadsPerThreadgroup = 64; + threadgroup float deltaShared[threadsPerThreadgroup]; + + uint height, width; + uint nbChannels; + uint nbThreadgroups; + uint nbBatch; + + if (pNbChannels && pDimensions && pNbThreadgroups && pNbBatch && + delta && outsPrev && squaredNorms && deltaTmp) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbThreadgroups = *pNbThreadgroups; + nbBatch = *pNbBatch; + } + else + return ; + + uint elem = id[1]; + uint remains = id[0]; + uint depth = remains / (height * width); + remains = remains % (height * width); + uint i = remains / width; + uint j = remains % width; + + if (depth * i * j >= nbChannels * height * width || + elem >= nbBatch) + { + return ; + } + + float norm = sqrt(squaredNorms[elem]); + if (norm > 1e-12) + { + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float deltaCur = delta[offset]; + float outPrev = outsPrev[offset]; + + deltaShared[threadId[0]] = outPrev * deltaCur; + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint stride=threadsPerThreadgroup/2; stride>0; stride>>=1) + { + uint index = threadId[0] + groupId[0] * threadsPerThreadgroup; + if (threadId[0] < stride && + (index + stride) < nbChannels * height * width) + { + deltaShared[threadId[0]] += deltaShared[threadId[0] + stride]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + if (threadId[0] == 0) + { + uint offset = elem * nbThreadgroups + groupId[0]; + deltaTmp[offset] = deltaShared[0]; + } + } +} + +kernel void normalize122DBackward( + const device float * delta, + const device float * outsPrev, + const device float * squaredNorms, + const device float * deltaTmp, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbThreadgroups, + constant uint * pNbBatch, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbThreadgroups; + uint nbBatch; + uint dirty; + + if (pNbChannels && pDimensions && pNbThreadgroups && pNbBatch && pDirty && + delta && outsPrev && squaredNorms && deltaTmp && deltaPrev) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbThreadgroups = *pNbThreadgroups; + nbBatch = *pNbBatch; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + float norm = sqrt(squaredNorms[elem]); + float deltaCurTmp = deltaTmp[elem]; + float normTmp = pow(norm, 3); + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float outPrev = outsPrev[offset]; + float deltaCur = delta[offset]; + + float newValue = 0.0; + if (norm > 1e-12) + { + newValue = deltaCur / norm - deltaCurTmp * outPrev / normTmp; + } + else + { + newValue = deltaCur / 1e-12; + } + + if (dirty) + { + deltaPrev[offset] = newValue; + } + else + { + deltaPrev[offset] += newValue; + } +} + +kernel void similarBatchError2DLoss( + const device float * outs, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + device float * losses, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbBatch; + + if (pNbChannels && pDimensions && pNbBatch && outs && losses) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + } + else + return ; + + uint elem1 = id[0]; + uint elem2 = id[1]; + + if (elem1 >= nbBatch || elem2 >= nbBatch) + { + return ; + } + + if (elem1 == elem2) + { + losses[elem2 + nbBatch * elem1] = 0.0; + } + else + { + float sum = 0.0; + for (uint i=0; i= width * height || elem >= nbBatch) + { + return ; + } + + float sum = 0.0; + for (uint elem1=0; elem1= width * height || elem >= nbBatchPrev) + { + return ; + } + + float sum = 0.0; + for (uint elem1=0; elem1= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset1 = j + (offsetStart + i) * width; + uint offset2 = offset1; + if (doFlip) + { + offset2 = width-1-j + (offsetStart + i) * width; + } + + outs[offset1] = outsPrev[offset2]; +} + +kernel void flipHorizontal2DBackward( + const device float * delta, + constant uint * pDoFlip, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint doFlip; + uint height, width; + uint nbChannels; + uint nbBatch; + uint dirty; + + if (pDoFlip && pNbChannels && pDimensions && pNbBatch && pDirty && + delta && deltaPrev) + { + doFlip = *pDoFlip; + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset1 = j + (offsetStart + i) * width; + uint offset2 = offset1; + if (doFlip) + { + offset2 = width-1-j + (offsetStart + i) * width; + } + + if (dirty) + { + deltaPrev[offset1] = delta[offset2]; + } + else + { + deltaPrev[offset1] += delta[offset2]; + } +} + +kernel void flipVertical2DForward( + const device float * outsPrev, + constant uint * pDoFlip, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint doFlip; + uint height, width; + uint nbChannels; + uint nbBatch; + + if (pDoFlip && pNbChannels && pDimensions && pNbBatch && + outsPrev && outs) + { + doFlip = *pDoFlip; + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset1 = j + (offsetStart + i) * width; + uint offset2 = offset1; + if (doFlip) + { + offset2 = j + (offsetStart + height-1-i) * width; + } + + outs[offset1] = outsPrev[offset2]; +} + +kernel void flipVertical2DBackward( + const device float * delta, + constant uint * pDoFlip, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint doFlip; + uint height, width; + uint nbChannels; + uint nbBatch; + uint dirty; + + if (pDoFlip && pNbChannels && pDimensions && pNbBatch && pDirty && + delta && deltaPrev) + { + doFlip = *pDoFlip; + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset1 = j + (offsetStart + i) * width; + uint offset2 = offset1; + if (doFlip) + { + offset2 = j + (offsetStart + height-1-i) * width; + } + + if (dirty) + { + deltaPrev[offset1] = delta[offset2]; + } + else + { + deltaPrev[offset1] += delta[offset2]; + } +} + +kernel void colorJitterHSVForward( + const device float * outsPrev, + constant float * pNoise, + constant uint * pDimensions, + constant uint * pNbBatch, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + float noiseH, noiseS, noiseV; + uint height, width; + uint nbBatch; + + if (pNoise && pDimensions && pNbBatch && outsPrev && outs) + { + noiseH = pNoise[0]; + noiseS = pNoise[1]; + noiseV = pNoise[2]; + width = pDimensions[0]; + height = pDimensions[1]; + nbBatch = *pNbBatch; + } + else + return ; + + uint elem = id[1]; + uint row = id[0] / width; + uint col = id[0] % width; + + if (row * col >= height * width || + elem >= nbBatch) + { + return ; + } + + uint offsetStartR = (0 + 3 * elem) * height; + uint offsetStartG = (1 + 3 * elem) * height; + uint offsetStartB = (2 + 3 * elem) * height; + + uint offsetR = col + (offsetStartR + row) * width; + uint offsetG = col + (offsetStartG + row) * width; + uint offsetB = col + (offsetStartB + row) * width; + + float r = outsPrev[offsetR]; + float g = outsPrev[offsetG]; + float b = outsPrev[offsetB]; + + float maxValue = max(max(r, g), b); + float minValue = min(min(r, g), b); + float delta = maxValue - minValue; + + float h; + if (delta == 0) + { + h = 0.0; + } + else if (maxValue == r) + { + h = (g - b) / delta; + } + else if (maxValue == g) + { + h = (g - b) / delta + 2.0; + } + else + { + h = (g - b) / delta + 4.0; + } + h *= 60.0; + + float s = 0.0; + if (maxValue != 0) + { + s = delta / maxValue; + } + + float v = maxValue; + + h += noiseH; h = max(h, 0.0); h = min(h, 360.0); + s += noiseS; s = max(s, 0.0); s = min(s, 1.0); + v += noiseV; v = max(v, 0.0); v = min(v, 1.0); + + if (s == 0.0) + { + r = v; g = v; b = v; + } + + float angle = h; + float sector = angle / 60; // Sector + float i = floor(sector); + float f = sector - i; // Factorial part of h + + float p = v * (1 - s); + float q = v * (1 - (s * f)); + float t = v * (1 - (s * (1 - f))); + + if (i == 0) + { + r = v; g = t; b = p; + } + else if (i == 1) + { + r = q; g = v; b = p; + } + else if (i == 2) + { + r = p; g = v; b = t; + } + else if (i == 3) + { + r = p; g = q; b = v; + } + else if (i == 4) + { + r = t; g = p; b = v; + } + else + { + r = v; g = p; b = q; + } + + outs[offsetR] = r; + outs[offsetG] = g; + outs[offsetB] = b; +} + +kernel void BCE2DLoss( + const device float * outs, + const device float * groundTruth, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + device float * losses, + uint id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbBatch; + + if (pNbChannels && pDimensions && pNbBatch && outs && groundTruth && losses) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + } + else + return ; + + uint elem = id; + if (elem >= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint depth=0; depth= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float gt = groundTruth[offset]; + float out = outs[offset]; + float derivative = 0.0; + + if (gt == 1.0) + { + derivative = -1 / out; + } + else if (gt == 0.0) + { + derivative = 1 / (1 - out); + } + + if (dirty) + { + deltaPrev[offset] = coeff * derivative / + float(nbBatch * nbChannels * height * width); + } + else + { + deltaPrev[offset] += coeff * derivative / + float(nbBatch * nbChannels * height * width); + } +} + +kernel void BCESigmoid2DLoss( + const device float * outs, + const device float * groundTruth, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pNbBatch, + device float * losses, + uint id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint nbBatch; + + if (pNbChannels && pDimensions && pNbBatch && outs && groundTruth && losses) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + nbBatch = *pNbBatch; + } + else + return ; + + uint elem = id; + if (elem >= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint depth=0; depth 0) + { + value = (1 - gt) * out; + value += log(1 + exp(-out)); + } + else + { + value = -out * gt; + value += log(exp(out) + 1); + } + + tmp += value; + }} + } + + losses[elem] = tmp; +} + +kernel void BCESigmoid2DLossDerivative( + const device float * outs, + const device float * groundTruth, + constant uint * pNbChannels, + constant uint * pDimensions, + constant float * pCoeff, + constant uint * pNbBatch, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + float coeff; + uint nbBatch; + uint dirty; + + if (pNbChannels && pDimensions && pNbBatch && pCoeff && pDirty && + outs && groundTruth && deltaPrev) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + coeff = *pCoeff; + nbBatch = *pNbBatch; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0] / width; + uint elem = id[1] / height; + uint i = id[1] % height; + uint j = id[0] % width; + + if (i * elem >= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + float gt = groundTruth[offset]; + float out = outs[offset]; + float value; + + if (out >= 0) + { + value = 1.0 / (1.0 + exp(-out)); + } + else + { + value = exp(out) / (1.0 + exp(out)); + } + + if (dirty) + { + deltaPrev[offset] = coeff * (value - gt) / + float(nbBatch * nbChannels * height * width); + } + else + { + deltaPrev[offset] += coeff * (value - gt) / + float(nbBatch * nbChannels * height * width); + } +} diff --git a/Sources/GrAIdient/Metal/Kernel/Merge.metal b/Sources/GrAIdient/Metal/Kernel/LayerMerge.metal similarity index 98% rename from Sources/GrAIdient/Metal/Kernel/Merge.metal rename to Sources/GrAIdient/Metal/Kernel/LayerMerge.metal index 59c8ef68..59e3db3c 100644 --- a/Sources/GrAIdient/Metal/Kernel/Merge.metal +++ b/Sources/GrAIdient/Metal/Kernel/LayerMerge.metal @@ -1,5 +1,5 @@ // -// Sum.metal +// LayerMerge.metal // GrAIdient // // Created by Jean-François Reboud on 14/10/2022. diff --git a/Sources/GrAIdient/Metal/Kernel/LayerNorm.metal b/Sources/GrAIdient/Metal/Kernel/LayerNorm.metal new file mode 100644 index 00000000..907b2602 --- /dev/null +++ b/Sources/GrAIdient/Metal/Kernel/LayerNorm.metal @@ -0,0 +1,294 @@ +// +// LayerNorm.metal +// GrAIdient +// +// Created by Jean-François Reboud on 09/03/2023. +// + +#include +using namespace metal; + +kernel void computeLayerNormSeqμ( + const device float * tmps, + constant uint * pNbNeurons, + constant uint * pNbBatch, + constant uint * pSequence, + device float * μ, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + uint sequence; + + if (pNbNeurons && pNbBatch && pSequence && tmps && μ) + { + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint elem = id[1]; + uint seq = id[0]; + if (elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint nbElems = nbNeurons; + float sum = 0.0; + + for (uint depth=0; depth= nbBatch || seq >= sequence) + { + return ; + } + + uint nbElems = nbNeurons; + float sum = 0.0; + + for (uint depth=0; depth= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint offset = depth + nbNeurons * seq + sequence * nbNeurons * elem; + + float tmp1 = tmps[offset] - μ[seq + sequence * elem]; + float tmp2 = sqrt(σ2[seq + sequence * elem] + Ɛ); + float xhat = tmp1 / tmp2; + xHat[offset] = xhat; + tmps[offset] = Ɣ[depth] * xhat + β[depth]; +} + +kernel void backwardWeights1LayerNormSeq( + const device float * delta, + const device float * xHat, + const device float * Ɣ, + constant uint * pNbNeurons, + constant uint * pNbBatch, + constant uint * pSequence, + device float * sum1, + device float * sum2, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + uint sequence; + + if (pNbNeurons && pNbBatch && pSequence && + delta && xHat && Ɣ && sum1 && sum2) + { + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint elem = id[1]; + uint seq = id[0]; + if (elem >= nbBatch || seq >= sequence) + { + return ; + } + + float tmp1 = 0.0, tmp2 = 0.0; + for (uint depth=0; depth= nbNeurons) + { + return ; + } + + float tmp1 = 0.0, tmp2 = 0.0; + for (uint elem=0; elem= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint offset = depth + nbNeurons * seq + sequence * nbNeurons * elem; + + float mult = + 1.0 / ((float)nbElems * sqrt(σ2[seq + sequence * elem] + Ɛ)); + float dxHat = Ɣ[depth] * delta[offset]; + float tmp1 = nbElems * dxHat; + float tmp2 = sum1[seq + sequence * elem]; + float tmp3 = xHat[offset] * sum2[seq + sequence * elem]; + + delta[offset] = mult * (tmp1 - tmp2 - tmp3); +} diff --git a/Sources/GrAIdient/Metal/Kernel/LayerSeq.metal b/Sources/GrAIdient/Metal/Kernel/LayerSeq.metal new file mode 100644 index 00000000..01d7d816 --- /dev/null +++ b/Sources/GrAIdient/Metal/Kernel/LayerSeq.metal @@ -0,0 +1,1028 @@ +// +// Layer1D.metal +// GrAIdient +// +// Created by Jean-François Reboud on 27/02/2023. +// + +#include +using namespace metal; + +kernel void avgPoolSeqForward( + const device float * outsPrev, + constant uint * pNbNeurons, + constant uint * pNbBatch, + constant uint * pSequence, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + uint sequence; + + if (pNbNeurons && pNbBatch && pSequence && + outsPrev && outs) + { + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1]; + + if (depth >= nbNeurons || elem >= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint seq=0; seq= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint offset = depth + nbNeurons * elem; + float deltaCur = delta[offset]; + + uint offsetPrev = depth + nbNeurons * seq + sequence * nbNeurons * elem; + if (dirty) + { + deltaPrev[offsetPrev] = deltaCur / sequence; + } + else + { + deltaPrev[offsetPrev] += deltaCur / sequence; + } +} + +kernel void selectSeqForward( + const device float * outsPrev, + constant uint * pNbNeurons, + constant uint * pTargetSeq, + constant uint * pNbBatch, + constant uint * pSequence, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint targetSeq; + uint nbNeurons; + uint nbBatch; + uint sequence; + + if (pTargetSeq && pNbNeurons && pNbBatch && pSequence && + outsPrev && outs) + { + targetSeq = *pTargetSeq; + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1]; + + if (depth >= nbNeurons || elem >= nbBatch) + { + return ; + } + + uint offset = depth + nbNeurons * elem; + uint offsetPrev = depth + + nbNeurons * targetSeq + sequence * nbNeurons * elem; + outs[offset] = outsPrev[offsetPrev]; +} + +kernel void selectSeqBackward( + const device float * delta, + constant uint * pNbNeurons, + constant uint * pTargetSeq, + constant uint * pNbBatch, + constant uint * pSequence, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + uint sequence; + uint targetSeq; + + if (pNbNeurons && pTargetSeq && pNbBatch && pSequence && + deltaPrev && delta) + { + targetSeq = *pTargetSeq; + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1]; + + if (depth >= nbNeurons || elem >= nbBatch) + { + return ; + } + + uint offset = depth + nbNeurons * elem; + uint offsetPrev = depth + + nbNeurons * targetSeq + sequence * nbNeurons * elem; + deltaPrev[offsetPrev] += delta[offset]; +} + +kernel void concat1SeqForward( + const device float * outsPrev, + constant uint * pGlobalOffset, + constant uint * pNbNeurons, + constant uint * pNbBatch, + constant uint * pSequence, + constant uint * pSequencePrev, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + uint sequence; + uint sequencePrev; + uint globalOffset; + + if (pGlobalOffset && pNbNeurons && + pNbBatch && pSequence && pSequencePrev && outsPrev && outs) + { + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + sequencePrev = *pSequencePrev; + globalOffset = *pGlobalOffset; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1] / sequencePrev; + uint seq = id[1] % sequencePrev; + + if (depth >= nbNeurons || elem >= nbBatch || seq >= sequencePrev) + { + return ; + } + + uint offsetPrev = depth + + nbNeurons * seq + sequencePrev * nbNeurons * elem; + uint offset = depth + + nbNeurons * (globalOffset+seq) + sequence * nbNeurons * elem; + + outs[offset] = outsPrev[offsetPrev]; +} + +kernel void concat1SeqBackward( + const device float * delta, + constant uint * pGlobalOffset, + constant uint * pNbNeurons, + constant uint * pNbBatch, + constant uint * pSequence, + constant uint * pSequencePrev, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + uint sequence; + uint sequencePrev; + uint globalOffset; + uint dirty; + + if (pGlobalOffset && pNbNeurons && + pNbBatch && pSequence && pSequencePrev && pDirty && deltaPrev && delta) + { + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + sequencePrev = *pSequencePrev; + globalOffset = *pGlobalOffset; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1] / sequencePrev; + uint seq = id[1] % sequencePrev; + + if (depth >= nbNeurons || elem >= nbBatch || seq >= sequencePrev) + { + return ; + } + + uint offsetPrev = depth + + nbNeurons * seq + sequencePrev * nbNeurons * elem; + uint offset = depth + + nbNeurons * (globalOffset+seq) + sequence * nbNeurons * elem; + + if (dirty) + { + deltaPrev[offsetPrev] = delta[offset]; + } + else + { + deltaPrev[offsetPrev] += delta[offset]; + } +} + +kernel void concat2SeqForward( + const device float * outsPrev, + constant uint * pGlobalOffset, + constant uint * pNbNeurons, + constant uint * pNbNeuronsPrev, + constant uint * pNbBatch, + constant uint * pSequence, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbNeuronsPrev; + uint nbBatch; + uint sequence; + uint globalOffset; + + if (pGlobalOffset && pNbNeurons && pNbNeuronsPrev && + pNbBatch && pSequence && outsPrev && outs) + { + nbNeurons = *pNbNeurons; + nbNeuronsPrev = *pNbNeuronsPrev; + nbBatch = *pNbBatch; + sequence = *pSequence; + globalOffset = *pGlobalOffset; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1] / sequence; + uint seq = id[1] % sequence; + + if (depth >= nbNeuronsPrev || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint offsetPrev = depth + + nbNeuronsPrev * seq + sequence * nbNeuronsPrev * elem; + uint offset = globalOffset+depth + + nbNeurons * seq + sequence * nbNeurons * elem; + + outs[offset] = outsPrev[offsetPrev]; +} + +kernel void concat2SeqBackward( + const device float * delta, + constant uint * pGlobalOffset, + constant uint * pNbNeurons, + constant uint * pNbNeuronsPrev, + constant uint * pNbBatch, + constant uint * pSequence, + constant uint * pDirty, + device float * deltaPrev, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbNeuronsPrev; + uint nbBatch; + uint sequence; + uint globalOffset; + uint dirty; + + if (pGlobalOffset && pNbNeurons && pNbNeuronsPrev && + pNbBatch && pSequence && pDirty && deltaPrev && delta) + { + nbNeurons = *pNbNeurons; + nbNeuronsPrev = *pNbNeuronsPrev; + nbBatch = *pNbBatch; + sequence = *pSequence; + globalOffset = *pGlobalOffset; + dirty = *pDirty; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1] / sequence; + uint seq = id[1] % sequence; + + if (depth >= nbNeuronsPrev || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint offsetPrev = depth + + nbNeuronsPrev * seq + sequence * nbNeuronsPrev * elem; + uint offset = globalOffset+depth + + nbNeurons * seq + sequence * nbNeurons * elem; + + if (dirty) + { + deltaPrev[offsetPrev] = delta[offset]; + } + else + { + deltaPrev[offsetPrev] += delta[offset]; + } +} + +kernel void constant12SeqForward( + const device float * weights, + constant uint * pNbNeurons, + constant uint * pNbBatch, + constant uint * pSequence, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + uint sequence; + + if (pNbNeurons && pNbBatch && pSequence && weights && outs) + { + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint depth = id[0]; + uint elem = id[1] / sequence; + uint seq = id[1] % sequence; + + if (depth >= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint offset = depth + nbNeurons * seq + sequence * nbNeurons * elem; + outs[offset] = weights[depth + nbNeurons * seq]; +} + +kernel void constant12SeqBackward( + const device float * delta, + constant uint * pNbNeurons, + constant uint * pNbBatch, + constant uint * pSequence, + constant uint * pAccumulate, + device float * grads, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint nbBatch; + uint sequence; + uint accumulate; + + if (pNbNeurons && pNbBatch && pSequence && pAccumulate && delta && grads) + { + nbNeurons = *pNbNeurons; + nbBatch = *pNbBatch; + sequence = *pSequence; + accumulate = *pAccumulate; + } + else + return ; + + uint depth = id[0]; + uint seq = id[1]; + if (depth >= nbNeurons || seq >= sequence) + { + return ; + } + + float tmp = 0.0; + for (uint elem=0; elem= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint offset = depth + nbNeurons * seq + sequence * nbNeurons * elem; + outs[offset] = weights[depth]; +} + +kernel void querySeqForward( + const device float * query, + const device float * key, + constant uint * pNbHeads, + constant uint * pNbNeurons, + constant uint * pNbNeuronsPrev, + constant uint * pNbBatch, + constant uint * pSequence, + device float * outs, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbHeads; + uint nbNeurons; + uint nbNeuronsPrev; + uint nbBatch; + uint sequence; + uint size; + + if (pNbHeads && pNbNeurons && pNbNeuronsPrev && pNbBatch && pSequence && + query && key && outs) + { + nbHeads = *pNbHeads; + nbNeurons = *pNbNeurons; + nbNeuronsPrev = *pNbNeuronsPrev; + nbBatch = *pNbBatch; + sequence = *pSequence; + size = nbNeuronsPrev / nbHeads; + } + else + return ; + + uint head = id[0] / sequence; + uint seqK = id[0] % sequence; + uint elem = id[1] / sequence; + uint seqQ = id[1] % sequence; + + if (head >= nbHeads || seqK >= sequence || + elem >= nbBatch || seqQ >= sequence) + { + return ; + } + + float tmp = 0.0; + for (uint j=0; j= nbHeads || j >= size || + elem >= nbBatch || seqQ >= sequence) + { + return ; + } + + float tmp = 0.0; + for (uint seqK=0; seqK= nbHeads || j >= size || + elem >= nbBatch || seqK >= sequence) + { + return ; + } + + float tmp = 0.0; + for (uint seqQ=0; seqQ= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + float cMax = outsPrev[ + 0+head*size + nbNeurons * seq + sequence * nbNeurons * elem + ]; + for (uint j=0; j cMax) + { + cMax = outPrev; + } + } + + float sum1 = 0.0; + for (uint j=0; j= nbNeurons || elem >= nbBatch || seq >= sequence) + { + return ; + } + + uint offset = depth + nbNeurons * seq + sequence * nbNeurons * elem; + float outCur = outs[offset]; + float deltaCur = delta[offset]; + + float sum1 = 0.0; + for (uint j=0; j= nbHeads || j >= size || + elem >= nbBatch || seqQ >= sequence) + { + return ; + } + + float tmp = 0.0; + for (uint seqK=0; seqK= nbHeads || j >= size || + elem >= nbBatch || seqK >= sequence) + { + return ; + } + + float tmp = 0.0; + for (uint seqQ=0; seqQ= nbHeads || seqK >= sequence || + elem >= nbBatch || seqQ >= sequence) + { + return ; + } + + float tmp = 0.0; + for (uint j=0; j +using namespace metal; + +kernel void reduce64( + const device float * ins, + constant uint * pDimensions, + constant uint * pNbThreadgroups, + device float * outs, + uint2 groupId [[ threadgroup_position_in_grid ]], + uint2 threadId [[ thread_position_in_threadgroup ]], + uint2 id [[ thread_position_in_grid ]]) +{ + constexpr uint threadsPerThreadgroup = 64; + threadgroup float sumShared[threadsPerThreadgroup]; + + uint dim1; + uint dim2; + uint nbThreadgroups; + + if (pDimensions && pNbThreadgroups && ins && outs) + { + dim1 = pDimensions[0]; + dim2 = pDimensions[1]; + nbThreadgroups = *pNbThreadgroups; + } + else + return ; + + uint elem1 = id[0]; + uint elem2 = id[1]; + + if (elem1 >= dim1 && elem2 >= dim2) + { + return ; + } + + uint offset = elem2 * dim1 + elem1; + sumShared[threadId[0]] = ins[offset]; + threadgroup_barrier(mem_flags::mem_threadgroup); + + for (uint stride=threadsPerThreadgroup/2; stride>0; stride>>=1) + { + uint index = threadId[0] + groupId[0] * threadsPerThreadgroup; + if (threadId[0] < stride && (index + stride) < dim1) + { + sumShared[threadId[0]] += sumShared[threadId[0] + stride]; + } + threadgroup_barrier(mem_flags::mem_threadgroup); + } + + if (threadId[0] == 0) + { + uint offset = elem2 * nbThreadgroups + groupId[0]; + outs[offset] = sumShared[0]; + } +} + +kernel void reduce( + const device float * ins, + constant uint * pDimensions, + device float * outs, + uint id [[ thread_position_in_grid ]]) +{ + uint dim1; + uint dim2; + + if (pDimensions && ins && outs) + { + dim1 = pDimensions[0]; + dim2 = pDimensions[1]; + } + else + return ; + + uint elem2 = id; + if (elem2 >= dim2) + { + return ; + } + + float sum = 0.0; + for (uint elem1=0; elem1 +using namespace metal; + +kernel void vq2DForward( + const device float * outsPrev, + const device float * weights, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pK, + constant uint * pNbBatch, + device float * outs, + device int * indices, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint K; + uint nbBatch; + + if (pNbChannels && pDimensions && pK && pNbBatch && + weights && outsPrev && outs && indices) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + K = *pK; + nbBatch = *pNbBatch; + } + else + return ; + + uint elem = id[1]; + uint i = id[0] / width; + uint j = id[0] % width; + + if (i * j >= height * width || elem >= nbBatch) + { + return ; + } + + int minIndex = -1; + float minValue = 0.0; + for (uint k=0; k= 0) + { + for (uint depth=0; depth= height * nbBatch || + j * depth >= width * nbChannels) + { + return ; + } + + uint offsetStart = (depth + nbChannels * elem) * height; + uint offset = j + (offsetStart + i) * width; + + int minIndex = indices[j + (elem * height + i) * width]; + uint offsetWeights = depth + nbChannels * minIndex; + + float vq = weights[offsetWeights]; + float deltaCur = delta[offset]; + float outPrev = outsPrev[offset]; + + if (dirty) + { + deltaPrev[offset] = deltaCur; + } + else + { + deltaPrev[offset] += deltaCur; + } + + // Commitment term. + deltaPrev[offset] += beta * 2.0 * (outPrev - vq); +} + +kernel void vq2DBatchDerWeights( + const device float * outsPrev, + const device float * weights, + const device int * indices, + constant uint * pNbChannels, + constant uint * pDimensions, + constant uint * pK, + constant float * pCoeff, + constant uint * pNbBatch, + device float * grads, + uint2 id [[ thread_position_in_grid ]]) +{ + uint height, width; + uint nbChannels; + uint K; + float coeff; + uint nbBatch; + + if (pNbChannels && pDimensions && pK && pCoeff && pNbBatch && + outsPrev && weights && indices && grads) + { + width = pDimensions[0]; + height = pDimensions[1]; + nbChannels = *pNbChannels; + K = *pK; + coeff = *pCoeff; + nbBatch = *pNbBatch; + } + else + return ; + + uint k = id[1]; + uint depth = id[0]; + + if (depth >= nbChannels || k >= K) + { + return ; + } + + float sum = 0.0; + for (uint elem=0; elem= nbChannels || elem * k >= nbBatch * K) + { + return ; + } + + float sum = 0.0; + for (uint i=0; i= nbChannels || k >= K) + { + return ; + } + + float sum = 0.0; + for (uint elem=0; elem= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint depth=0; depth +using namespace metal; + +kernel void vqSeqForward( + const device float * outsPrev, + const device float * weights, + constant uint * pNbNeurons, + constant uint * pK, + constant uint * pNbBatch, + constant uint * pSequence, + device float * outs, + device int * indices, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint K; + uint nbBatch; + uint sequence; + + if (pNbNeurons && pK && pNbBatch && pSequence && + weights && outsPrev && outs && indices) + { + nbNeurons = *pNbNeurons; + K = *pK; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint elem = id[1]; + uint seq = id[0]; + + if (seq >= sequence || elem >= nbBatch) + { + return ; + } + + int minIndex = -1; + float minValue = 0.0; + for (uint k=0; k= 0) + { + for (uint depth=0; depth= sequence * nbBatch || + depth >= nbNeurons) + { + return ; + } + + uint offset = depth + nbNeurons * seq + sequence * nbNeurons * elem; + + int minIndex = indices[seq + elem * sequence]; + uint offsetWeights = depth + nbNeurons * minIndex; + + float vq = weights[offsetWeights]; + float deltaCur = delta[offset]; + float outPrev = outsPrev[offset]; + + if (dirty) + { + deltaPrev[offset] = deltaCur; + } + else + { + deltaPrev[offset] += deltaCur; + } + + // Commitment term. + deltaPrev[offset] += beta * 2.0 * (outPrev - vq); +} + +kernel void vqSeqBatchDerWeights( + const device float * outsPrev, + const device float * weights, + const device int * indices, + constant uint * pNbNeurons, + constant uint * pK, + constant float * pCoeff, + constant uint * pNbBatch, + constant uint * pSequence, + device float * grads, + uint2 id [[ thread_position_in_grid ]]) +{ + uint nbNeurons; + uint K; + float coeff; + uint nbBatch; + uint sequence; + + if (pNbNeurons && pK && pCoeff && pNbBatch && pSequence && + outsPrev && weights && indices && grads) + { + nbNeurons = *pNbNeurons; + K = *pK; + coeff = *pCoeff; + nbBatch = *pNbBatch; + sequence = *pSequence; + } + else + return ; + + uint k = id[1]; + uint depth = id[0]; + + if (depth >= nbNeurons || k >= K) + { + return ; + } + + float sum = 0.0; + for (uint elem=0; elem= nbNeurons || elem * k >= nbBatch * K) + { + return ; + } + + float sum = 0.0; + for (uint seq=0; seq= nbBatch) + { + return ; + } + + float tmp = 0.0; + for (uint depth=0; depth self.deviceID = deviceID self.nbElems = nbElems } + + /// + /// Download the content of the buffer to the CPU. + /// + /// - Returns: the CPU buffer. + /// + public func download() -> UnsafeMutableBufferPointer + { + fatalError("Not implemented.") + } + + /// Upload the content of the buffer to the GPU. + public func upload() + { + fatalError("Not implemented.") + } } /// @@ -77,6 +93,23 @@ public class MetalPrivateBuffer: MetalBuffer return _shared! } } + + /// + /// Download the content of the buffer to the CPU. + /// + /// - Returns: the CPU buffer. + /// + public override func download() -> UnsafeMutableBufferPointer + { + MetalKernel.get.download([self]) + return shared.buffer + } + + /// Upload the content of the buffer to the GPU. + public override func upload() + { + MetalKernel.get.upload([self]) + } } /// @@ -140,9 +173,31 @@ public class MetalSharedBuffer: MetalBuffer super.init(nbElems, deviceID: deviceID) } + /// + /// Free memory. + /// + /// This is necessary as memory was initialized with a specific method. + /// deinit { free(memory) } + + /// + /// Download the content of the buffer to the CPU. + /// + /// - Returns: the CPU buffer. + /// + public override func download() -> UnsafeMutableBufferPointer + { + MetalKernel.get.download([self]) + return buffer + } + + /// Upload the content of the buffer to the GPU. + public override func upload() + { + MetalKernel.get.upload([self]) + } } public extension UnsafeMutableRawPointer diff --git a/Sources/GrAIdient/Metal/MetalConfig.swift b/Sources/GrAIdient/Metal/MetalConfig.swift new file mode 100644 index 00000000..e0985c24 --- /dev/null +++ b/Sources/GrAIdient/Metal/MetalConfig.swift @@ -0,0 +1,227 @@ +// +// MetalConfig.swift +// GrAIdient +// +// Created by Jean-François Reboud on 18/05/2023. +// + +let CONFIG_KERNELS = +[ + "Activation": [ + "forwardReLU", + "backwardReLU", + "forwardLeakyReLU", + "backwardLeakyReLU", + "forwardSoftReLU", + "backwardSoftReLU", + "forwardSigmoid", + "backwardSigmoid", + "forwardGELU", + "backwardGELU", + ], + "Biases": [ + "reduceBiases", + ], + "BatchNorm": [ + "computeBNConvμ", + "computeBNConvσ2", + "forwardBNConvTraining", + "forwardBNConvInference", + "backwardWeightsBNConv", + "backwardBNConvTraining", + "backwardBNConvInference", + ], + "Convolution": [ + "convForward", + "convBackward", + "convBatchDerWeights", + "convBatchDerBiases", + "convDerWeights", + "convDerBiases", + "convReduceWeights", + ], + "Deconvolution": [ + "deconvForward", + "deconvBackward", + "deconvBatchDerWeights", + "deconvDerWeights", + ], + "FullyConnected": [ + "flForward", + "flBackward", + "flBatchDerWeights", + "flBatchDerBiases", + "flDerWeights", + "flDerBiases", + "flReduceWeights", + ], + "FullyConnectedPatch": [ + "flPatchForward", + "flPatchBackward", + "flPatchBatchDerWeights", + "flPatchBatchDerBiases", + "flPatchDerWeights", + "flPatchDerBiases", + "flPatchReduceWeights", + ], + "FullyConnectedSeq": [ + "flSeqForward", + "flSeqBackward", + "flSeqBatchDerWeights", + "flSeqDerWeights", + "flSeqReduceWeights", + ], + "InstanceNorm": [ + "computeInstanceNormConvμ", + "computeInstanceNormConvσ2", + "forwardInstanceNormConv", + "forwardAdaIN", + "backwardWeightsInstanceNormConv", + "backward2AdaIN", + "backwardInstanceNormConv", + "backward1AdaIN", + ], + "Layer1D": [ + "MSE1DLoss", + "MSE1DLossDerivative", + "linearErrorLoss", + "linearErrorLossDerivative", + "selectNeurons1DForward", + "selectNeurons1DBackward", + "concat1DForward", + "concat1DBackward", + "softmax1DForward", + "softmax1DBackward", + "dotProduct1DForward", + "dotProduct1DBackward", + "constant1DForward", + "BCE1DLoss", + "BCE1DLossDerivative", + "BCESigmoid1DLoss", + "BCESigmoid1DLossDerivative", + ], + "Layer2D": [ + "avgPoolForward", + "avgPoolBackward", + "maxPoolForward", + "maxPoolBackward", + "adaptiveAvgPoolForward1", + "adaptiveAvgPoolForward2", + "adaptiveAvgPoolBackward1", + "adaptiveAvgPoolBackward2", + "selectNeurons2DForward", + "selectNeurons2DBackward", + "IRDFT2RGBForward", + "IRDFT2RGBBackward", + "decorrelateRGBForward", + "decorrelateRGBBackward", + "linearScale2DForward", + "linearScale2DBackward", + "setDataFTFrequences2D", + "pad2DForward", + "pad2DBackward", + "crop2DForward", + "crop2DBackward", + "resizeBilinearPadForward", + "resizeBilinearPadBackward", + "rotate2DForward", + "rotate2DBackward", + "resizeBilinearCropForward", + "resizeBilinearCropBackward", + "concat02DForward", + "concat02DBackward", + "concat12DForward", + "concat12DBackward", + "constant2DForward", + "MSE2DLoss", + "MSE2DLossDerivative", + "selfCorrelate2DForward", + "selfCorrelate2DBackward", + "normalize12DForward", + "normalize12DBackward", + "computeSquaredNorm122D", + "normalize122DForward", + "computeDeltaTmp122D", + "normalize122DBackward", + "similarBatchError2DLoss", + "similarBatchError2DLossDerivative", + "similarError2DLossDerivative", + "flipHorizontal2DForward", + "flipHorizontal2DBackward", + "flipVertical2DForward", + "flipVertical2DBackward", + "colorJitterHSVForward", + "BCE2DLoss", + "BCE2DLossDerivative", + "BCESigmoid2DLoss", + "BCESigmoid2DLossDerivative", + ], + "LayerMerge": [ + "sum1", + "sum2", + "multiplyForward", + "multiplyBackward", + ], + "LayerNorm": [ + "computeLayerNormSeqμ", + "computeLayerNormSeqσ2", + "forwardLayerNormSeq", + "backwardWeights1LayerNormSeq", + "backwardWeights2LayerNormSeq", + "backwardLayerNormSeq", + ], + "LayerSeq": [ + "avgPoolSeqForward", + "avgPoolSeqBackward", + "concat1SeqForward", + "concat1SeqBackward", + "concat2SeqForward", + "concat2SeqBackward", + "constant12SeqForward", + "constant12SeqBackward", + "constant2SeqForward", + "querySeqForward", + "queryQuerySeqBackward", + "queryKeySeqBackward", + "softmaxSeqForward", + "softmaxSeqBackward", + "valueSeqForward", + "valueValueSeqBackward", + "valueScoreSeqBackward", + "selectSeqForward", + "selectSeqBackward", + ], + "Optimizer": [ + "clipGradients", + "multiplyGradients", + "weightsSGD", + "weightsMomentum", + "weightsAdam", + "weightsAMSGrad", + "weightsAdamRectified", + "weightsAdaBound", + "weightsAMSBound", + ], + "Reduce": [ + "reduce64", + "reduce", + ], + "Reset": [ + "reset" + ], + "VQ2D": [ + "vq2DForward", + "vq2DBackward", + "vq2DBatchDerWeights", + "vq2DDerWeights", + "vq2DReduceWeights", + "vq2DLoss" + ], + "VQSeq": [ + "vqSeqForward", + "vqSeqBackward", + "vqSeqBatchDerWeights", + "vqSeqDerWeights", + "vqSeqLoss" + ] +] diff --git a/Sources/GrAIdient/Metal/MetalKernel.swift b/Sources/GrAIdient/Metal/MetalKernel.swift index 491b8978..7228653c 100644 --- a/Sources/GrAIdient/Metal/MetalKernel.swift +++ b/Sources/GrAIdient/Metal/MetalKernel.swift @@ -527,123 +527,7 @@ private class MetalDevice /// Initialize the GPU kernels' state. private func _initKernels() { - let listKernels = - [ - "Activation": [ - "forwardReLU", - "backwardReLU", - "forwardLeakyReLU", - "backwardLeakyReLU", - "forwardSoftReLU", - "backwardSoftReLU", - "forwardSigmoid", - "backwardSigmoid", - ], - "Biases": [ - "reduceBiases", - ], - "BN": [ - "computeConvμ", - "computeConvσ2", - "forwardBNConvTraining", - "forwardBNConvInference", - "backwardWeightsBNConv", - "backwardBNConvTraining", - "backwardBNConvInference", - ], - "Convolution": [ - "convForward", - "convBackward", - "convBatchDerWeights", - "convBatchDerBiases", - "convDerWeights", - "convDerBiases", - "convReduceWeights", - ], - "Deconvolution": [ - "deconvForward", - "deconvBackward", - "deconvBatchDerWeights", - "deconvDerWeights", - ], - "FullyConnected": [ - "flForward", - "flBackward", - "flBatchDerWeights", - "flBatchDerBiases", - "flDerWeights", - "flDerBiases", - "flReduceWeights", - ], - "Layer1D": [ - "MSE1DLoss", - "MSE1DLossDerivative", - "linearErrorLoss", - "linearErrorLossDerivative", - "selectNeurons1DForward", - "selectNeurons1DBackward", - "concat1DForward", - "concat1DBackward", - "softmax1DForward", - "softmax1DBackward", - "dotProduct1DForward", - "dotProduct1DBackward", - "constant1DForward", - ], - "Layer2D": [ - "avgPoolForward", - "avgPoolBackward", - "maxPoolForward", - "maxPoolBackward", - "adaptiveAvgPoolForward1", - "adaptiveAvgPoolForward2", - "adaptiveAvgPoolBackward1", - "adaptiveAvgPoolBackward2", - "selectNeurons2DForward", - "selectNeurons2DBackward", - "IRDFT2RGBForward", - "IRDFT2RGBBackward", - "decorrelateRGBForward", - "decorrelateRGBBackward", - "linearScale2DForward", - "linearScale2DBackward", - "setDataFTFrequences2D", - "pad2DForward", - "pad2DBackward", - "crop2DForward", - "crop2DBackward", - "resizeBilinearPadForward", - "resizeBilinearPadBackward", - "rotate2DForward", - "rotate2DBackward", - "resizeBilinearCropForward", - "resizeBilinearCropBackward", - "concat2DForward", - "concat2DBackward", - ], - "Merge": [ - "sum1", - "sum2", - "multiplyForward", - "multiplyBackward", - ], - "Optimizer": [ - "clipGradients", - "multiplyGradients", - "weightsSGD", - "weightsMomentum", - "weightsAdam", - "weightsAMSGrad", - "weightsAdamRectified", - "weightsAdaBound", - "weightsAMSBound", - ], - "Reset": [ - "reset" - ] - ] - - for (libName, kernelNames) in listKernels + for (libName, kernelNames) in CONFIG_KERNELS { let lib = Bundle.module.url( forResource: libName, diff --git a/Sources/GrAIdient/Metal/Reduce.swift b/Sources/GrAIdient/Metal/Reduce.swift new file mode 100644 index 00000000..aa2e9984 --- /dev/null +++ b/Sources/GrAIdient/Metal/Reduce.swift @@ -0,0 +1,99 @@ +// +// Reduce.swift +// GrAIdient +// +// Created by Jean-François Reboud on 17/05/2023. +// + +import MetalKit + +/// Default number of threads per threadgroup on the GPU. +private let THREADS_PER_THREADGROUP = 64 + +/// +/// Compute the number of threadgroups resulting in the reduction of `nbElems` on the GPU. +/// +/// - Parameters: +/// - nbElems: The number of elements to reduce. +/// - threadsPerThreadroup: The number of threads per thread group. +/// - Returns: The number of resulting threadgroups. +/// +private func getNbThreadgroups( + nbElems: Int, + threadsPerThreadgroup: Int) -> Int +{ + let value = Double(nbElems) / Double(threadsPerThreadgroup) + return Int(ceil(value)) +} + +/// +/// Reduce (sum) the elements in the `dim1` dimension. +/// +/// - Parameters: +/// - inBuffer: The input buffer. +/// - outBuffer: The final buffer (also used for writing intermediate results). +/// - dim1: The dimension of the elements to reduce. +/// - dim2: A dimension for elements we do not want to reduce. +/// +public func reduce( + inBuffer: MTLBuffer, + outBuffer: MTLBuffer, + dim1: Int, + dim2: Int, + deviceID: Int) +{ + var nbElems = dim1 + var command: MetalCommand + + while nbElems > 1 + { + let pNbDimensions: [UInt32] = [UInt32(nbElems), UInt32(dim2)] + + // Reduce thanks to thread group shared memory. + if nbElems > 100 + { + let nbThreadgroups = getNbThreadgroups( + nbElems: nbElems, + threadsPerThreadgroup: THREADS_PER_THREADGROUP + ) + let pNbThreadgroups: [UInt32] = [UInt32(nbThreadgroups)] + + command = MetalKernel.get.createCommand( + "reduce64", deviceID: deviceID + ) + command.setBuffer(inBuffer, atIndex: 0) + command.setBytes(pNbDimensions, atIndex: 1) + command.setBytes(pNbThreadgroups, atIndex: 2) + command.setBuffer(outBuffer, atIndex: 3) + + let threadsPerThreadgroup = MTLSizeMake( + THREADS_PER_THREADGROUP, 1, 1 + ) + let threadsPerGrid = MTLSizeMake( + nbElems, dim2, 1 + ) + command.dispatchThreads( + threadsPerGrid: threadsPerGrid, + threadsPerThreadgroup: threadsPerThreadgroup + ) + + nbElems = nbThreadgroups + } + + // Simple reduce. + else + { + command = MetalKernel.get.createCommand( + "reduce", deviceID: deviceID + ) + command.setBuffer(inBuffer, atIndex: 0) + command.setBytes(pNbDimensions, atIndex: 1) + command.setBuffer(outBuffer, atIndex: 2) + + command.dispatchThreads(dim2) + + nbElems = 1 + } + command.enqueue() + } +} diff --git a/Sources/GrAIdient/Utils/Image.swift b/Sources/GrAIdient/Utils/Image.swift new file mode 100644 index 00000000..2450a321 --- /dev/null +++ b/Sources/GrAIdient/Utils/Image.swift @@ -0,0 +1,424 @@ +// +// Image.swift +// GrAIdient +// +// Created by Jean-François Reboud on 19/05/2023. +// + +import Foundation +import Cocoa + +/// Error occuring when processing images. +public enum ImageError: Error +{ + /// Problem in the dimensions of the image. + case UnexpectedSize + /// Impossible to get access to the image's pixels. + case UnexpectedDataStructure + /// Not enough space in a buffer. + case MissingSpace +} + +extension ImageError: CustomStringConvertible +{ + public var description: String + { + switch self + { + case .UnexpectedSize: + return "Size is too big." + case .UnexpectedDataStructure: + return "Cannot extract pixels." + case .MissingSpace: + return "Not enough space." + } + } +} + +public class Image +{ + /// + /// Load images into a buffer of pixels. + /// + /// Consider the input images are in the .RGB `ImageFormat` and + /// the output buffer in the .Neuron format. + /// + /// - Parameters: + /// - metalBuffer: Buffer of images. + /// - width: Width of the images. + /// - height: Height of the images. + /// - Returns: The list of images as list of pixels. + /// + public static func loadImages( + imagesURL: [URL], + imagesBuffer: MetalBuffer, + width: Int, + height: Int) throws + { + let batchSize = imagesURL.count + if imagesBuffer.nbElems < batchSize * 3 * height * width + { + throw ImageError.MissingSpace + } + + let bufferPtr = imagesBuffer.download() + for (elem, imageURL) in imagesURL.enumerated() + { + let image = NSImage(contentsOfFile: imageURL.path)! + let pixels = try image.extractPaddedPixels( + width: CGFloat(width), height: CGFloat(height) + ) + + for i in 0.., + width: Int, + height: Int) -> [[UInt8]] + { + let bufferPtr = metalBuffer.download() + let nbImages = metalBuffer.nbElems / (width * height * 3) + + var output = [[UInt8]]() + for elem in 0.. 255.0 + { + val = 255 + } + else + { + val = UInt8(valTmp) + } + + gridPtr[3 * offsetSet + depth] = val + }} + output.append(grid) + } + return output + } + + /// + /// Convert float images (values in [0; 1]) to pixel images (values in [0; 255]). + /// + /// - Parameter images: List of images with values in [0; 1]. + /// - Returns: The list of images with values in [0; 255]. + /// + public static func toPixel( + _ images: [[T]]) -> [[UInt8]] + { + var output = [[UInt8]]() + for elem in 0.. 255.0 + { + val = 255 + } + else + { + val = UInt8(valTmp) + } + return val + }) + } + return output + } + + /// + /// Convert pixel images (values in [0; 255]) to float images (values in [0; 1]). + /// + /// - Parameter images: List of images with values in [0; 255]. + /// - Returns: The list of images with values in [0; 1]. + /// + public static func toFloat( + _ images: [[UInt8]]) -> [[T]] + { + var output = [[T]]() + for elem in 0.. [[UInt8]] + { + var output = [[UInt8]]() + for elem in 0.. [[UInt8]] + { + var output = [[UInt8]]() + for elem in 0.. NSImage + { + let bitsPerComponent = 8 + let bitsPerPixel = 24 + + let rgbColorSpace = CGColorSpaceCreateDeviceRGB() + let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.none.rawValue) + + if (pixels.count != Int(3 * width * height)) + { + fatalError() + } + + var data = pixels // Copy to mutable [] + let providerRef = CGDataProvider( + data: Data( + bytes: &data, + count: data.count * MemoryLayout.size + ) as CFData + ) + + let cgImage = CGImage( + width: width, + height: height, + bitsPerComponent: bitsPerComponent, + bitsPerPixel: bitsPerPixel, + bytesPerRow: 3 * width * MemoryLayout.size, + space: rgbColorSpace, + bitmapInfo: bitmapInfo, + provider: providerRef!, + decode: nil, + shouldInterpolate: true, + intent: CGColorRenderingIntent.defaultIntent + )! + return NSImage(cgImage: cgImage, size: NSZeroSize) + } +} + +public extension NSImage +{ + /// + /// Extract the underlying pixels in the image. + /// Pad with zeros when needed. + /// + /// Throw an error when image is too big or it is + /// impossible to retrieve the pixels. + /// + /// - Parameters: + /// - width: The expected output image width. + /// - height: The expected output image height. + /// - Returns: An array of pixels. + /// + func extractPaddedPixels( + width: CGFloat, + height: CGFloat) throws -> [UInt8] + { + let pixelsIn = try extractPixels() + + if self.size.height == height && + self.size.width == width + { + return pixelsIn + } + else if self.size.height < height || + self.size.width < width + { + var pixelsOut = [UInt8]( + repeating: 0, count: Int(3 * width * height) + ) + let offsetIn = 3 * Int(size.width) + let offsetOut = 3 * Int(width) + for i in 0.. [UInt8] + { + if let imageData = tiffRepresentation, + let imageRep = NSBitmapImageRep(data: imageData), + let dataPtr = imageRep.bitmapData + { + let bufferPtr = UnsafeBufferPointer( + start: dataPtr, + count: Int(3 * size.height * size.width) + ) + return [UInt8](bufferPtr) + } + else + { + throw ImageError.UnexpectedDataStructure + } + } + + /// + /// Save an image to the disk. + /// + /// Throw an error in the Cocoa domain, if there is an error writing to the `URL`. + /// + /// - Parameter url: The path where to dump the image. + /// + func save(url: URL) throws + { + if representations.count > 0 { + if let imageData = tiffRepresentation + { + let rep = NSBitmapImageRep(data: imageData)! + let pngData = rep.representation( + using: NSBitmapImageRep.FileType.png, + properties: [:] + )! + try pngData.write(to: url, options: []) + }} + } +} diff --git a/Sources/GrAIdient/Utils/Serialization.swift b/Sources/GrAIdient/Utils/Serialization.swift index 6e9c4e37..42593625 100644 --- a/Sources/GrAIdient/Utils/Serialization.swift +++ b/Sources/GrAIdient/Utils/Serialization.swift @@ -30,36 +30,68 @@ let LAYER_REGISTRY: [String: Codable.Type] = buildRegistry( [ Activation1D.self, Activation2D.self, + AdaIN.self, AdaptiveAvgPool2D.self, AvgPool2D.self, + AvgPoolSeq.self, + BCE1D.self, + BCE2D.self, + BCESigmoid1D.self, + BCESigmoid2D.self, BN2D.self, + ColorJitterHSV.self, Concat1D.self, Concat2D.self, + Concat1Seq.self, + Concat2Seq.self, Constant1D.self, + Constant2D.self, + Constant12Seq.self, + Constant2Seq.self, Convolution2D.self, Crop2D.self, Deconvolution2D.self, DecorrelateRGB.self, DotProduct1D.self, + FlipHorizontal2D.self, + FlipVertical2D.self, FTFrequences2D.self, FullyConnected.self, + FullyConnectedPatch.self, + FullyConnectedSeq.self, Input1D.self, Input2D.self, IRDFT2RGB.self, + InstanceNorm2D.self, + LayerNormSeq.self, LinearError1D.self, LinearScale2D.self, MaxPool2D.self, + Normalize12D.self, + Normalize122D.self, MSE1D.self, + MSE2D.self, Multiply2D.self, Pad2D.self, + QuerySeq.self, + ResizeBilinear.self, ResizeBilinearCrop.self, ResizeBilinearPad.self, Rotate2D.self, + SelfCorrelate2D.self, Softmax1D.self, + SoftmaxSeq.self, Sum1D.self, Sum2D.self, + SumSeq.self, SelectNeurons1D.self, SelectNeurons2D.self, + SelectSeq.self, + SimilarityBatchError2D.self, + SimilarityError2D.self, + ValueSeq.self, + VQ2D.self, + VQSeq.self, ]) /// diff --git a/Tests/GrAIExamples/AutoEncoderExample.swift b/Tests/GrAIExamples/AutoEncoderExample.swift new file mode 100644 index 00000000..f2daa9a3 --- /dev/null +++ b/Tests/GrAIExamples/AutoEncoderExample.swift @@ -0,0 +1,385 @@ +// +// AutoEncoderExample.swift +// GrAIExamples +// +// Created by Aurélien PEDEN on 23/03/2023. +// Modified by Jean-François Reboud on 21/05/2023. +// + +import XCTest +import GrAIdient + +/// Train a simple Auto Encoder model on the CIFAR dataset. +final class AutoEncoderExample: XCTestCase +{ + /// Directory to dump outputs from the tests. + let _outputDir = NSTemporaryDirectory() + + /// Batch size of data. + let _batchSize = 16 + + /// Initialize test. + override func setUp() + { + setPythonLib() + _ = MetalKernel.get + GrAI.Opti.GPU = true + } + + /// + /// Get optimizer parameters for model training. + /// + /// - Parameter nbLoops: Number of steps per epoch. + /// - Returns: The optimizer parameters. + /// + func _getOptimizerParams(nbLoops: Int) -> GrAI.Optimizer.Params + { + var optimizerParams = GrAI.Optimizer.Params() + optimizerParams.nbLoops = nbLoops + + // Simple optimizer scheduler: always the same optimizer during + // the training. + optimizerParams.optimizer = ConstEpochsScheduler( + GrAI.Optimizer.Class.AdamRectified + ) + + // Simple variable scheduler: always the same variable during + // the training. + optimizerParams.variables["alpha"] = ConstEpochsVar( + value: ConstVal(1e-3) + ) + optimizerParams.variables["lambda"] = ConstEpochsVar( + value: ConstVal(1e-6) + ) + + // Other schedulers can be built thanks to `GrAI.Optimizer.Params`. + return optimizerParams + } + + /// + /// Build an encoder branch with `nbBlock` blocks of dimension reduction (factor of 2). + /// + /// - Parameters: + /// - size: Size of one image (height and width are the same) after resize. + /// - nbBlocks: Number of reduction blocks. + /// - params: Contextual parameters linking to the model. + /// - Returns: A list of layers at different image resolutions. + /// + func _buildEncoder( + size: Int, + nbBlocks: Int, + params: GrAI.Model.Params) -> [Layer2D] + { + var layer: Layer2D + var layers = [Layer2D]() + + layer = Input2D( + nbChannels: 3, + width: size, height: size, + params: params + ) + + for _ in 0.. Layer2D + { + var layer: Layer2D = layersPrev.first! + var numLayer = 0 + + while numLayer < layersPrev.count + { + layer = Deconvolution2D( + layerPrev: layer, size: 2, nbChannels: 8, stride: 2, + activation: nil, biases: true, bn: false, + params: params + ) + + if numLayer + 1 < layersPrev.count + { + layer = try! Concat2D( + layersPrev: [layersPrev[numLayer + 1], layer], + params: params + ) + layer = Convolution2D( + layerPrev: layer, + size: 3, nbChannels: 8, stride: 1, + activation: ReLU.str, biases: true, bn: false, + params: params + ) + } + else + { + layer = Convolution2D( + layerPrev: layer, + size: 3, nbChannels: 3, stride: 1, + activation: Sigmoid.str, biases: true, bn: false, + params: params + ) + } + numLayer += 1 + } + return layer + } + + /// + /// Build a style controller branch. + /// + /// - Parameters: + /// - layersPrev: A tuple of layers at different image resolutions. + /// - params: Contextual parameters linking to the model. + /// - Returns: The last layer of the style branch. + /// + func _buildStyleMapping( + layersPrev: [Layer2D], + params: GrAI.Model.Params) -> Layer1D + { + var layers = [Layer1D]() + for layerPrev in layersPrev + { + layers.append( + AvgPool2D(layerPrev: layerPrev, params: params) + ) + } + var layer: Layer1D = Concat1D( + layersPrev: layers, params: params + ) + for _ in 0..<8 + { + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 8, + activation: ReLU.str, biases: true, + params: params + ) + } + return layer + } + + /// + /// Build a StyleGAN like decoder branch with `nbBlock` blocks + /// of dimension augmentation (factor of 2). + /// + /// - Parameters: + /// - nbBlocks: Number of augmentation blocks. + /// - style: The last layer of the style branch. + /// - params: Contextual parameters linking to the model. + /// - Returns: The last layer of the decoder branch. + /// + func _buildStyleDecoder( + nbBlocks: Int, + style: Layer1D, + params: GrAI.Model.Params) -> Layer2D + { + var layer: Layer2D + layer = Constant2D( + nbChannels: 8, height: 2, width: 2, + params: params + ) + layer = try! AdaIN( + layersPrev: [ + layer, + FullyConnected( + layerPrev: style, nbNeurons: 2 * 8, + activation: ReLU.str, biases: true, + params: params + ) + ], + params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 3, nbChannels: 8, stride: 1, + activation: ReLU.str, biases: true, bn: false, + params: params + ) + layer = try! AdaIN( + layersPrev: [ + layer, + FullyConnected( + layerPrev: style, nbNeurons: 2 * 8, + activation: ReLU.str, biases: true, + params: params + ) + ], + params: params + ) + + for _ in 0.. Model + { + // Create the context to build a graph of layers where + // there is no previous model dependency: layer id starts at 0. + let context = ModelContext(name: "AutoEncoder", models: []) + let params = GrAI.Model.Params(context: context) + + let layersPrev = _buildEncoder( + size: size, + nbBlocks: nbBlocks, + params: params + ) + + switch modelType + { + case .Style: + _ = _buildStyleDecoder( + nbBlocks: nbBlocks, + style: _buildStyleMapping( + layersPrev: layersPrev, + params: params + ), + params: params + ) + case .UNet: + _ = _buildUNetDecoder( + layersPrev: layersPrev, + params: params + ) + } + return Model(model: context.model, modelsPrev: []) + } + + /// + /// Train the model. + /// + /// - Parameters: + /// - model: The model to train. + /// - size: Size of one image (height and width are the same) after resize. + /// + func _trainModel(model: Model, size: Int) + { + let trainer = try! CIFARAutoEncoderTrainer( + model: model, size: size + ) + trainer.run( + batchSize: _batchSize, + label: 8, + nbEpochs: 2, + keep: 1000 + ) + } + + /// Test1: train a simple auto encoder model. + func test1_TrainSimpleModel() + { + let size = 32 + + // Build a model with randomly initialized weights. + let model = SimpleAutoEncoder.build(size) + + // Train model. + _trainModel( + model: model, + size: size + ) + } + + /// Test2: train a UNet like auto encoder model. + func test2_TrainUNetModel() + { + let nbBlocks = 5 + let size = min(Int(pow(2.0, Double(nbBlocks))), 32) + + // Build a model with randomly initialized weights. + let model = _buildModel( + modelType: .UNet, + size: size, + nbBlocks: nbBlocks + ) + + // Train model. + _trainModel( + model: model, + size: size + ) + } + + /// Test3: train a StyleGAN like auto encoder model. + func test3_TrainStyleModel() + { + let nbBlocks = 5 + let size = min(Int(pow(2.0, Double(nbBlocks))), 32) + + // Build a model with randomly initialized weights. + let model = _buildModel( + modelType: .Style, + size: size, + nbBlocks: nbBlocks + ) + + // Train model. + _trainModel( + model: model, + size: size + ) + } +} diff --git a/Tests/GrAIExamples/AutoEncoderTests.swift b/Tests/GrAIExamples/AutoEncoderTests.swift new file mode 100644 index 00000000..f1850f90 --- /dev/null +++ b/Tests/GrAIExamples/AutoEncoderTests.swift @@ -0,0 +1,63 @@ +// +// AutoEncoderTests.swift +// GrAIExamples +// +// Created by Jean-François Reboud on 21/05/2023. +// + +import XCTest +import PythonKit +import GrAIdient + +/// Compare models trained in GrAIdient and PyTorch. +final class AutoEncoderTests: XCTestCase +{ + /// Batch size of data. + let _batchSize = 16 + + /// The label associated to the data. + let _label = 8 + + /// Size of one image (height and width are the same). + let _size = 32 + + /// Initialize test. + override func setUp() + { + setPythonLib() + _ = MetalKernel.get + GrAI.Opti.GPU = true + } + + /// Compare loss in the training of a simple auto encoder model in GrAIdient and PyTorch. + func testTrain() throws + { + let trainer1 = try CIFARAutoEncoderTrainer( + model: SimpleAutoEncoder.build(_size), + size: _size + ) + trainer1.initTrain( + batchSize: _batchSize, + label: _label + ) + + let pythonLib = Python.import("python_lib") + let trainer2 = pythonLib.train_simple_auto_encoder( + _batchSize, + _label + ) + + for _ in 0..<100 + { + let computedLoss = trainer1.step() + let expectedLoss = Float( + pythonLib.step_simple_auto_encoder(trainer2) + )! + + // Compare difference. + let diffPercent = + abs(computedLoss - expectedLoss) / abs(expectedLoss) * 100.0 + XCTAssert(diffPercent < 0.1) + } + } +} diff --git a/Tests/GrAIExamples/Base/CIFAR.swift b/Tests/GrAIExamples/Base/CIFAR.swift index 9ba01ba3..f5c2fcec 100644 --- a/Tests/GrAIExamples/Base/CIFAR.swift +++ b/Tests/GrAIExamples/Base/CIFAR.swift @@ -51,7 +51,7 @@ class CIFAR: DataSamplerImpl var dataset = [UInt8]() for dataFile in 1...5 { - let data = pythonLib.load_CIFAR_data(dataFile, label, size) + let data = pythonLib.load_CIFAR_train(dataFile, label, size) dataset += Array(data)! } @@ -108,5 +108,41 @@ class CIFAR: DataSamplerImpl } return CIFAR(data: dataset, size: size) } + + /// + /// Build an iterator on CIFAR dataset. + /// + /// - Parameters: + /// - train: Train of test dataset. + /// - batchSize: The batch size. + /// - label: The label we want the data associated to. + /// - shuffle: Whether to shuffle indices of data. + /// + /// - Returns: A Python iterator. + /// + static func buildIterator( + train: Bool, + batchSize: Int, + label: Int, + shuffle: Bool) -> PythonObject + { + let pythonLib = Python.import("python_lib") + return pythonLib.iter_CIFAR(train, batchSize, label, shuffle) + } + + /// + /// Load next data from a Python iterator. + /// + /// - Parameter iterator: The Python iterator. + /// + static func getSamples(_ iterator: PythonObject) -> ([Float], Int) + { + let pythonLib = Python.import("python_lib") + let data = pythonLib.next_data_CIFAR(iterator) + + let samples = [Float](data.tuple2.0)! + let batchSize = Int(data.tuple2.1)! + + return (samples, batchSize) + } } - diff --git a/Tests/GrAIExamples/Base/CIFARAutoEncoderTrainer.swift b/Tests/GrAIExamples/Base/CIFARAutoEncoderTrainer.swift new file mode 100644 index 00000000..000398d6 --- /dev/null +++ b/Tests/GrAIExamples/Base/CIFARAutoEncoderTrainer.swift @@ -0,0 +1,366 @@ +// +// CIFARAutoEncoderTrainer.swift +// GrAIExamples +// +// Created by Jean-François Reboud on 21/05/2023. +// + +import Foundation +import GrAIdient + +/// Error occuring when trainer cannot be built. +public enum TrainerError: Error +{ + /// Model size is not coherent. + case Size + /// Model structure is not expected. + case Structural +} + +extension TrainerError: CustomStringConvertible +{ + public var description: String + { + switch self + { + case .Size: + return "Model size is not coherent." + case .Structural: + return "Model first layer should be an Input2D." + } + } +} + +/// Train an auto encoder model on CIFAR dataset. +class CIFARAutoEncoderTrainer +{ + /// Directory to dump outputs from the tests. + let _outputDir = NSTemporaryDirectory() + + /// Size of one image (height and width are the same) in the CIFAR datasset. + let _originalSize = 32 + /// Size of one image (height and width are the same) after resize. + let _size: Int + + /// Mean of the preprocessing to apply to data. + let _mean: (Float, Float, Float) = (123.675, 116.28, 103.53) + /// Deviation of the preprocessing to apply to data. + let _std: (Float, Float, Float) = (58.395, 57.12, 57.375) + + /// Dataset to get the data from. + var _dataset: CIFAR! = nil + /// Final model that is being trained. + var _model: Model! = nil + /// Resizer model. + var _resizer: Model? = nil + /// Base model to train. + let _baseModel: Model + + /// + /// Create the trainer. + /// + /// `size` allows to simulate the fact that the model analyzes a coarse image: the inputs and + /// ground truths are resized to `size` in order to do so. + /// + /// Throw an error if the original model's first layer is not an `Input2D` or the size of the latter + /// is not the size expected by the trainer. + /// + /// - Parameters: + /// - model: The original model (auto encoder structure) to train. + /// - size: Size of one image (height and width are the same). + /// + init(model: Model, size: Int) throws + { + _size = size + + if size > _originalSize || size < 2 + { + throw TrainerError.Size + } + + guard let firstLayer = model.layers.first as? Input2D else + { + throw TrainerError.Structural + } + + let height = firstLayer.height + let width = firstLayer.width + if height != _size || width != _size + { + throw TrainerError.Size + } + + _baseModel = model + } + + /// + /// Create the final model (containing the original one + some additional layers) to train. + /// + /// - Returns: The final model to train. + /// + private func _buildModel() -> Model + { + let context = ModelContext(name: "Final", models: [_baseModel]) + let params = GrAI.Model.Params(context: context) + + _ = try! MSE2D( + layerPrev: _baseModel.layers.last as! Layer2D, + params: params + ) + + var model = Model(name: "Final") + model.layers = _baseModel.layers + context.model.layers + model = Model(model: model, modelsPrev: []) + + return model + } + + /// + /// Create a resizer. + /// + /// - Returns: The resizer model. + /// + private func _buildResizer() -> Model? + { + if _size != _originalSize + { + let context = ModelContext(name: "Resizer", models: []) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 3, + width: _originalSize, + height: _originalSize, + params: params + ) + layer = try! ResizeBilinear( + layerPrev: layer, + dimension: _size, + params: params + ) + return Model(model: context.model, modelsPrev: []) + } + else + { + return nil + } + } + + /// + /// Get optimizer parameters for model training. + /// + /// - Parameter nbLoops: Number of steps per epoch. + /// - Returns: The optimizer parameters. + /// + func _getOptimizerParams(nbLoops: Int) -> GrAI.Optimizer.Params + { + var optimizerParams = GrAI.Optimizer.Params() + optimizerParams.nbLoops = nbLoops + + // Simple optimizer scheduler: always the same optimizer during + // the training. + optimizerParams.optimizer = ConstEpochsScheduler( + GrAI.Optimizer.Class.Adam + ) + + // Simple variable scheduler: always the same variable during + // the training. + optimizerParams.variables["alpha"] = ConstEpochsVar( + value: ConstVal(1e-3) + ) + optimizerParams.variables["lambda"] = ConstEpochsVar( + value: ConstVal(1e-6) + ) + + // Other schedulers can be built thanks to `GrAI.Optimizer.Params`. + return optimizerParams + } + + /// + /// Initialize dataset, model and optimizer parameters. + /// + /// - Parameters: + /// - batchSize: The number of samples per batch of data. + /// - label: The class of the CIFAR dataset to use. + /// - keep: The number of elements to keep in the dataset. + /// + func initTrain(batchSize: Int, label: Int, keep: Int? = nil) + { + // Create dataset. + CIFAR.dumpTrain( + datasetPath: _outputDir + "/datasetTrain\(label)", + label: label, + size: _originalSize + ) + + // Load dataset. + _dataset = CIFAR.loadDataset( + datasetPath: _outputDir + "/datasetTrain\(label)", + size: _originalSize + ) + _dataset.initSamples(batchSize: batchSize) + if let nbElems = keep + { + _dataset.keep(nbElems) + } + + // Get optimizer parameters for iterating over batch size elements. + let params = _getOptimizerParams(nbLoops: batchSize) + + // Build model. + _model = _buildModel() + + // Build resizer model. + _resizer = _buildResizer() + + // Initialize for training. + _model.initialize(params: params, phase: .Training) + _resizer?.initKernel() + } + + /// + /// One training step. + /// + /// - Returns: The loss on the last training step. + /// + func step() -> Float + { + let firstLayer: Input2D = _model.layers.first as! Input2D + let lastLayer: MSE2D = _model.layers.last as! MSE2D + + // Get data. + let samples = _dataset.getSamples()! + let batchSize = samples.count + + // Pre processing. + let data = preprocess( + samples, + height: _originalSize, + width: _originalSize, + mean: _mean, + std: _std, + imageFormat: .Neuron + ) + + // Reset gradient validity for backward pass + // and update the batch size. + _model.updateKernel(batchSize: batchSize) + + let dataLayer: Layer2D + // Resize data when `_size` is lower than `_originalSize`. + if let resizer = _resizer + { + let resizerFirstLayer = resizer.layers.first as! Input2D + dataLayer = resizer.layers.last as! Layer2D + + resizer.updateKernel(batchSize: batchSize) + + // Set data. + try! resizerFirstLayer.setDataGPU( + data, + batchSize: batchSize, + nbChannels: 3, height: _originalSize, width: _originalSize, + format: .Neuron + ) + + // Forward. + try! resizer.forward() + + // Set resized data. + try! firstLayer.setDataGPU( + dataLayer.outs, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size + ) + } + else + { + // Set data. + try! firstLayer.setDataGPU( + data, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size, + format: .Neuron + ) + dataLayer = firstLayer + } + + // Forward. + try! _model.forward() + + // Apply loss derivative: take into account the potential coarse image. + try! lastLayer.lossDerivativeGPU( + dataLayer.outs, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size + ) + + // Backward. + try! _model.backward() + + // Update weights. + try! _model.update() + + // Get loss result. + // Note that backward is explicitly + // enabled by `applyGradient` whereas `getLoss` is + // just an indicator. + let loss = try! lastLayer.getLossGPU( + dataLayer.outs, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size + ) + + // Update internal step. + // This is not mandatory except if we used another + // optimizer scheduler: see `_getOptimizerParams`. + _model.incStep() + + return loss + } + + /// + /// Run the training on multiple steps and multiple epochs. + /// + /// - Parameters: + /// - batchSize: The number of samples per batch of data. + /// - label: The class of the CIFAR dataset to use. + /// - nbEpochs: The number of epochs for the training to continue. + /// - keep: The number of elements to keep in the dataset. + /// + func run(batchSize: Int, label: Int, nbEpochs: Int, keep: Int? = nil) + { + initTrain( + batchSize: batchSize, + label: label, + keep: keep + ) + + for epoch in 0.. Model + { + let context = ModelContext(name: "SimpleAutoEncoder", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: size, + height: size, + params: params + ) + + layer = Convolution2D( + layerPrev: layer, + size: 3, nbChannels: 12, stride: 2, + activation: ReLU.str, biases: true, bn: false, + params: params + ) + layer = Convolution2D( + layerPrev: layer, + size: 3, nbChannels: 24, stride: 2, + activation: ReLU.str, biases: true, bn: false, + params: params + ) + layer = Convolution2D( + layerPrev: layer, + size: 3, nbChannels: 48, stride: 2, + activation: ReLU.str, biases: true, bn: false, + params: params + ) + + layer = Deconvolution2D( + layerPrev: layer, + size: 2, nbChannels: 24, stride: 2, + activation: nil, biases: true, bn: false, + params: params + ) + layer = Deconvolution2D( + layerPrev: layer, + size: 2, nbChannels: 12, stride: 2, + activation: nil, biases: true, bn: false, + params: params + ) + layer = Deconvolution2D( + layerPrev: layer, + size: 2, nbChannels: 3, stride: 2, + activation: Sigmoid.str, biases: true, bn: false, + params: params + ) + + let model = Model(model: context.model, modelsPrev: []) + + // Load weights from `PyTorch`. + let pythonLib = Python.import("python_lib") + let data = pythonLib.load_simple_auto_encoder_weights() + + let weights = [[Float]](data.tuple2.0)! + + // Apply weights on the `GrAIdient` model's layers. + var cur = 0 + for num_layer in 0.., - width: Int, - height: Int) -> [[UInt8]] -{ - let bufferPtr: UnsafeMutableBufferPointer - if let sBuffer = metalBuffer as? MetalSharedBuffer - { - MetalKernel.get.download([sBuffer]) - bufferPtr = sBuffer.buffer - } - else if let pBuffer = metalBuffer as? MetalPrivateBuffer - { - MetalKernel.get.download([pBuffer]) - bufferPtr = pBuffer.shared.buffer - } - else - { - fatalError() - } - - let nbImages = metalBuffer.nbElems / (width * height * 3) - - var output = [[UInt8]]() - for elem in 0.. 255.0 - { - val = 255 - } - else - { - val = UInt8(valTmp) - } - - gridPtr[3 * offsetSet + depth] = val - }} - output.append(grid) - } - return output -} - -/// -/// Get pixels out of images. -/// -/// - Parameters: -/// - images: List of images. -/// - width: Width of the images. -/// - height: Height of the images. -/// - imageFormat: The image format. -/// - Returns: The list of images as list of pixels. -/// -func getPixels( - _ images: [[UInt8]], - width: Int, - height: Int, - imageFormat: ImageFormat) -> [[UInt8]] -{ - switch imageFormat - { - case .RGB: - return images - - case .Neuron: - var output = [[UInt8]]() - for elem in 0.. NSImage -{ - let bitsPerComponent = 8 - let bitsPerPixel = 24 - - let rgbColorSpace = CGColorSpaceCreateDeviceRGB() - let bitmapInfo = CGBitmapInfo(rawValue: CGImageAlphaInfo.none.rawValue) - - if (pixels.count != Int(3 * width * height)) - { - fatalError() - } - - var data = pixels // Copy to mutable [] - let providerRef = CGDataProvider( - data: Data( - bytes: &data, - count: data.count * MemoryLayout.size - ) as CFData - ) - - let cgImage = CGImage( - width: width, - height: height, - bitsPerComponent: bitsPerComponent, - bitsPerPixel: bitsPerPixel, - bytesPerRow: 3 * width * MemoryLayout.size, - space: rgbColorSpace, - bitmapInfo: bitmapInfo, - provider: providerRef!, - decode: nil, - shouldInterpolate: true, - intent: CGColorRenderingIntent.defaultIntent - )! - return NSImage(cgImage: cgImage, size: NSZeroSize) -} - -/// -/// Save an image to the disk. -/// -/// - Parameters: -/// - image: The image to save to the disk. -/// - url: The path where to dump the image. -/// -func saveImage(_ image: NSImage, url: URL) -{ - if image.representations.count > 0 { - if let imageData = image.tiffRepresentation - { - let rep = NSBitmapImageRep(data: imageData)! - let pngData = rep.representation( - using: NSBitmapImageRep.FileType.png, - properties: [:] - )! - try! pngData.write(to: url, options: []) - }} -} diff --git a/Tests/GrAIExamples/Base/python_lib/__init__.py b/Tests/GrAIExamples/Base/python_lib/__init__.py index a0dcf991..04ee09bd 100644 --- a/Tests/GrAIExamples/Base/python_lib/__init__.py +++ b/Tests/GrAIExamples/Base/python_lib/__init__.py @@ -1,6 +1,23 @@ -from python_lib.cifar import load_CIFAR_data, load_CIFAR_test +from python_lib.cifar import ( + load_CIFAR_train, + load_CIFAR_test, + iter_CIFAR, + next_data_CIFAR, +) +from python_lib.weight import ( + load_simple_auto_encoder_weights, +) +from python_lib.trainer import ( + train_simple_auto_encoder, + step_simple_auto_encoder, +) __all__ = [ - "load_CIFAR_data", + "load_CIFAR_train", "load_CIFAR_test", + "iter_CIFAR", + "next_data_CIFAR", + "load_simple_auto_encoder_weights", + "train_simple_auto_encoder", + "step_simple_auto_encoder", ] diff --git a/Tests/GrAIExamples/Base/python_lib/cifar.py b/Tests/GrAIExamples/Base/python_lib/cifar.py index 785fe2cd..ac89724a 100644 --- a/Tests/GrAIExamples/Base/python_lib/cifar.py +++ b/Tests/GrAIExamples/Base/python_lib/cifar.py @@ -1,8 +1,15 @@ import cv2 import pickle +import torch +import torchvision import numpy as np -from typing import List from pathlib import Path +from typing import Tuple, List, Optional +from torchvision.transforms import ( + ToTensor, + Normalize, + Compose +) def extract_images( @@ -50,7 +57,7 @@ def extract_images( return ret_images[label] -def load_CIFAR_data( +def load_CIFAR_train( data_file: int, label: int, size: int @@ -73,7 +80,8 @@ def load_CIFAR_data( The list of flatten images with inner shape: (batch, channel, height, width). """ - data_dir = Path(__file__).parent.parent.resolve() / "data" / "in" + data_dir = Path(__file__).parent.parent.parent.parent.resolve() / \ + "data" / "in" / "cifar-10-batches-py" with open(f"{data_dir}/data_batch_{data_file}", 'rb') as fo: dict = pickle.load(fo, encoding='bytes') @@ -101,9 +109,119 @@ def load_CIFAR_test( The list of flatten images with inner shape: (batch, channel, height, width). """ - data_dir = Path(__file__).parent.parent.resolve() / "data" / "in" + data_dir = Path(__file__).parent.parent.parent.parent.resolve() / \ + "data" / "in" / "cifar-10-batches-py" with open(f"{data_dir}/test_batch", 'rb') as fo: dict = pickle.load(fo, encoding='bytes') return extract_images(data_dict=dict, label=label, size=size) + + +class MaskSampler(torch.utils.data.sampler.Sampler): + """ + Sampler of indices that is based on a mask. + + Parameters + ---------- + mask: np.ndarray + Base mask of the indices to consider. + """ + + def __init__(self, mask: np.ndarray): + self.indices = np.nonzero(mask)[0] + + def __iter__(self): + return iter(self.indices) + + def __len__(self): + return len(self.indices) + + +def iter_CIFAR( + train: bool, + batch_size: int, + label: int, + shuffle: bool +): + """ + Build an iterator on CIFAR dataset. + + Parameters + ---------- + train: bool + Train or test dataset. + batch_size: int + The batch size. + label: int + The label we want the data associated to. + shuffle: bool + Whether to shuffle indices of data. + + Returns + ------- + An iterator on CIFAR dataset. + """ + mean = (0.485, 0.456, 0.406) + std = (0.229, 0.224, 0.225) + transform = Compose([ + ToTensor(), + Normalize(mean, std) + ]) + data_dir = Path(__file__).parent.parent.parent.parent.resolve() / \ + "data" / "in" + cifar = torchvision.datasets.CIFAR10( + root=data_dir, train=train, download=True, transform=transform + ) + indices = np.array(cifar.targets) == label + + return iter(torch.utils.data.DataLoader( + cifar, batch_size=batch_size, shuffle=shuffle, num_workers=0, + sampler=MaskSampler(indices) + )) + + +def next_tensor_CIFAR(iterator) -> Optional[torch.Tensor]: + """ + Load next data from a CIFAR iterator. + + Parameters + ---------- + iterator + The CIFAR dataset iterator. + + Returns + ------- + torch.Tensor + The images tensor with inner shape: + (batch, channel, height, width). + """ + try: + samples, _ = next(iterator) + except StopIteration: + return None + return samples + + +def next_data_CIFAR(iterator) -> Tuple[List[float], int]: + """ + Load and flatten next data from a CIFAR iterator. + + Parameters + ---------- + iterator + The CIFAR dataset iterator. + + Returns + ------- + List[int] + The list of flatten images with inner shape: + (batch, channel, height, width). + int + The batch size of data. + """ + samples = next_tensor_CIFAR(iterator) + if samples is not None: + return samples.flatten().tolist(), len(samples) + else: + return [], 0 diff --git a/Tests/GrAIExamples/Base/python_lib/model.py b/Tests/GrAIExamples/Base/python_lib/model.py new file mode 100644 index 00000000..f3753138 --- /dev/null +++ b/Tests/GrAIExamples/Base/python_lib/model.py @@ -0,0 +1,72 @@ +import torch + + +class SimpleAutoEncoder(torch.nn.Module): + """ + Simple auto encoder model. + """ + + def __init__(self): + super().__init__() + self.encoder = torch.nn.Sequential( + torch.nn.Conv2d( + 3, 12, + kernel_size=3, stride=2, padding=1, + bias=True + ), + torch.nn.ReLU(), + torch.nn.Conv2d( + 12, 24, + kernel_size=3, stride=2, padding=1, + bias=True + ), + torch.nn.ReLU(), + torch.nn.Conv2d( + 24, 48, + kernel_size=3, stride=2, padding=1, + bias=True + ), + torch.nn.ReLU(), + ) + self.decoder = torch.nn.Sequential( + torch.nn.ConvTranspose2d(48, 24, kernel_size=2, stride=2), + torch.nn.ConvTranspose2d(24, 12, kernel_size=2, stride=2), + torch.nn.ConvTranspose2d(12, 3, kernel_size=2, stride=2), + torch.nn.Sigmoid(), + ) + + self.encoder.apply(self.weight_init) + self.decoder.apply(self.weight_init) + + @staticmethod + def weight_init(module: torch.nn.Module): + """ + Initialize weights and biases. + + Parameters + ---------- + module: torch.nn.Module + The module to initialize. + """ + if isinstance(module, torch.nn.Conv2d) or \ + isinstance(module, torch.nn.ConvTranspose2d) or \ + isinstance(module, torch.nn.Linear): + torch.nn.init.xavier_normal_(module.weight) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass. + + Parameters + ---------- + x: torch.Tensor + The input tensor. + + Returns + ------- + _: torch.Tensor + The output tensor. + """ + x = self.encoder(x) + x = self.decoder(x) + return x diff --git a/Tests/GrAIExamples/Base/python_lib/trainer.py b/Tests/GrAIExamples/Base/python_lib/trainer.py new file mode 100644 index 00000000..4a91aeca --- /dev/null +++ b/Tests/GrAIExamples/Base/python_lib/trainer.py @@ -0,0 +1,72 @@ +import torch +from typing import Optional + +from python_lib.cifar import ( + iter_CIFAR, + next_tensor_CIFAR, +) +from python_lib.model import SimpleAutoEncoder + + +def train_simple_auto_encoder( + batch_size: int, + label: int +): + """ + Build a simple auto encoder trainer. + + Parameters + ---------- + batch_size: int + The batch size. + label: int + The label we want the data associated to. + + Returns + ------- + A trainer on a simple auto encoder model. + """ + torch.manual_seed(42) + model = SimpleAutoEncoder().cpu() + + criterion = torch.nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=0.001) + + iter_data = iter_CIFAR( + train=True, + batch_size=batch_size, + label=label, + shuffle=False + ) + + while True: + samples = next_tensor_CIFAR(iter_data) + x = model(samples) + loss = criterion(x, samples) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + yield float(loss.detach().numpy()) + + +def step_simple_auto_encoder(trainer) -> Optional[float]: + """ + Compute next loss from the simple auto encoder trainer. + + Parameters + ---------- + trainer + The auto encoder trainer. + + Returns + ------- + float + The loss computed. + """ + try: + loss = next(trainer) + except StopIteration: + return None + return loss diff --git a/Tests/GrAIExamples/Base/python_lib/weight.py b/Tests/GrAIExamples/Base/python_lib/weight.py new file mode 100644 index 00000000..18698b40 --- /dev/null +++ b/Tests/GrAIExamples/Base/python_lib/weight.py @@ -0,0 +1,96 @@ +import torch +import numpy as np +from typing import List, Tuple + +from python_lib.model import SimpleAutoEncoder + + +def _flatten_weights( + weights: np.ndarray +) -> Tuple[List[float], List[int]]: + """ + Flatten weights and biases. + + Parameters + ---------- + weights: np.ndarray + The weights to flatten. + + Returns + ------- + (_, _): List[float], List[int] + The flattened weights, their shape. + """ + weights_list = weights.flatten().tolist() + dims_list = list(weights.shape) + + return weights_list, dims_list + + +def _extract_and_transpose_weights( + modules: [torch.nn.Module] +) -> Tuple[List[List[float]], List[List[int]]]: + """ + Get weights and biases. + Transpose weights when they come from a + ConvTranspose2d layer. + + Parameters + ---------- + modules: [torch.nn.Module] + The list of modules to get the weights and biases from. + + Returns + ------- + (_, _): List[List[float]], List[List[int]] + The flattened weights, their shape. + """ + layers_weights: List[List[float]] = [] + layers_dims: List[List[int]] = [] + for module in modules: + submodules = list(module.children()) + if len(submodules) > 0: + (weights_list, dims_list) = _extract_and_transpose_weights( + submodules + ) + layers_weights += weights_list + layers_dims += dims_list + + else: + if hasattr(module, "weight"): + if isinstance(module, torch.nn.ConvTranspose2d): + weights = np.transpose( + module.weight.detach().numpy(), (1, 0, 2, 3) + ) + weights_list, dims_list = _flatten_weights(weights) + + else: + weights = module.weight.detach().numpy() + weights_list, dims_list = _flatten_weights(weights) + + layers_weights.append(weights_list) + layers_dims.append(dims_list) + + if hasattr(module, "bias"): + weights = module.bias.detach().numpy() + weights_list, dims_list = _flatten_weights(weights) + + layers_weights.append(weights_list) + layers_dims.append(dims_list) + + return layers_weights, layers_dims + + +def load_simple_auto_encoder_weights( +) -> Tuple[List[List[float]], List[List[int]]]: + """ + Get weights and biases for simple auto encoder model. + + Returns + ------- + (_, _): List[List[float]], List[List[int]] + The flattened weights, their shape. + """ + torch.manual_seed(42) + model = SimpleAutoEncoder() + return _extract_and_transpose_weights(list(model.children())) diff --git a/Tests/GrAIExamples/Base/setup.py b/Tests/GrAIExamples/Base/setup.py index ee5f51d9..ca515733 100644 --- a/Tests/GrAIExamples/Base/setup.py +++ b/Tests/GrAIExamples/Base/setup.py @@ -7,6 +7,8 @@ author='Jean-François Reboud', license='MIT', install_requires=[ + "torch==1.10.1", + "torchvision==0.11.2", "numpy==1.23.1", "opencv-python==4.6.0.66" ], diff --git a/Tests/GrAIExamples/CIFARTests.swift b/Tests/GrAIExamples/CIFARTests.swift index 2a8ea985..5fd7bc9a 100644 --- a/Tests/GrAIExamples/CIFARTests.swift +++ b/Tests/GrAIExamples/CIFARTests.swift @@ -18,6 +18,11 @@ final class CIFARTests: XCTestCase /// Size of one image (height and width are the same). let _size = 32 + /// Mean of the preprocessing to apply to data. + let _mean: (Float, Float, Float) = (123.675, 116.28, 103.53) + /// Deviation of the preprocessing to apply to data. + let _std: (Float, Float, Float) = (58.395, 57.12, 57.375) + /// Initialize test. override func setUp() { @@ -108,7 +113,7 @@ final class CIFARTests: XCTestCase XCTAssert(nbLoops == cifar.nbLoops) } - /// Test4: dump testing dataset and load it.. + /// Test4: dump testing dataset and load it. func test4_DumpTest() { let datasetPath = _outputDir + "/datasetTest" @@ -122,4 +127,66 @@ final class CIFARTests: XCTestCase size: _size ) } + + /// Test5: iterate on CIFAR, preprocess and compare with PyTorch results. + func test5_PreprocessSamples() + { + let cifar = CIFAR.loadDataset( + datasetPath: _outputDir + "/datasetTrain", + size: _size + ) + cifar.initSamples(batchSize: _batchSize) + + let iterator = CIFAR.buildIterator( + train: true, + batchSize: _batchSize, + label: 0, + shuffle: false + ) + + var nbLoops = 0 + var lastLoop = false + var batchSize = 0 + var samples2 = [Float]() + + while let samples1 = cifar.getSamples() + { + (samples2, batchSize) = CIFAR.getSamples(iterator) + + XCTAssert(!lastLoop) + if samples1.count != _batchSize + { + lastLoop = true + } + else + { + XCTAssert(samples1.count == _batchSize) + XCTAssert(batchSize == _batchSize) + } + + // Pre processing. + let data: [Float] = preprocess( + samples1, + height: _size, + width: _size, + mean: _mean, + std: _std, + imageFormat: .Neuron + ) + + for (elem1, elem2) in zip(data, samples2) + { + XCTAssertEqual(elem1, elem2, accuracy: 0.0001) + } + nbLoops += 1 + } + + print("Number of loops per epoch: " + String(nbLoops)) + XCTAssert(nbLoops == cifar.nbLoops) + XCTAssert(cifar.getSamples() == nil) + + (samples2, batchSize) = CIFAR.getSamples(iterator) + XCTAssert(samples2.count == 0) + XCTAssert(batchSize == 0) + } } diff --git a/Tests/GrAIExamples/TransformerExample.swift b/Tests/GrAIExamples/TransformerExample.swift new file mode 100644 index 00000000..d281622f --- /dev/null +++ b/Tests/GrAIExamples/TransformerExample.swift @@ -0,0 +1,385 @@ +// +// TransformerExample.swift +// GrAIExamples +// +// Created by Aurélien PEDEN on 14/03/2023. +// + +import XCTest +import GrAIdient + +/// Train a simple Vision Transformer model on the CIFAR dataset. +final class TransformerExample: XCTestCase +{ + /// Directory to dump outputs from the tests. + let _outputDir = NSTemporaryDirectory() + + /// Batch size of data. + let _batchSize = 64 + /// Size of one image (height and width are the same). + let _size = 32 + + /// Mean of the preprocessing to apply to data. + let _mean: (Float, Float, Float) = (123.675, 116.28, 103.53) + /// Deviation of the preprocessing to apply to data. + let _std: (Float, Float, Float) = (58.395, 57.12, 57.375) + + // Initialize test. + override func setUp() + { + setPythonLib() + _ = MetalKernel.get + GrAI.Opti.GPU = true + } + + /// + /// Get optimizer parameters for model training. + /// + /// - Parameter nbLoops: Number of steps per epoch. + /// - Returns: The optimizer parameters. + /// + func _getOptimizerParams(nbLoops: Int) -> GrAI.Optimizer.Params + { + var optimizerParams = GrAI.Optimizer.Params() + optimizerParams.nbLoops = nbLoops + + // Simple optimizer scheduler: always the same optimizer during + // the training. + optimizerParams.optimizer = ConstEpochsScheduler( + GrAI.Optimizer.Class.AdamRectified + ) + + // Simple variable scheduler: always the same variable during + // the training. + optimizerParams.variables["alpha"] = ConstEpochsVar( + value: ConstVal(1e-3) + ) + optimizerParams.variables["lambda"] = ConstEpochsVar( + value: ConstVal(1e-6) + ) + + // Other schedulers can be built thanks to `GrAI.Optimizer.Params`. + return optimizerParams + } + + /// + /// Build a multi attention branch. + /// + /// - Parameters: + /// - layerPrev: previous layer. + /// - nbHeads: Number of head in attention branches. + /// - hiddenDim: Dimension of neurons in the main branch. + /// - params: Contextual parameters linking to the model. + /// - Returns: The last layer of the branch. + /// + func _buildMultiHeadAttention( + layerPrev: LayerSeq, + nbHeads: Int, + hiddenDim: Int, + params: GrAI.Model.Params) -> LayerSeq + { + let query: LayerSeq = FullyConnectedSeq( + layerPrev: layerPrev, nbNeurons: hiddenDim, + activation: nil, biases: true, + params: params + ) + let key: LayerSeq = FullyConnectedSeq( + layerPrev: layerPrev, nbNeurons: hiddenDim, + activation: nil, biases: true, + params: params + ) + let value: LayerSeq = FullyConnectedSeq( + layerPrev: layerPrev, nbNeurons: hiddenDim, + activation: nil, biases: true, + params: params + ) + + var layerSeq: LayerSeq = try! QuerySeq( + query: query, key: key, nbHeads: nbHeads, + params: params + ) + layerSeq = try! SoftmaxSeq( + layerPrev: layerSeq, nbHeads: nbHeads, + params: params + ) + + layerSeq = try! ValueSeq( + value: value, score: layerSeq, nbHeads: nbHeads, + params: params + ) + + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: hiddenDim, + activation: nil, biases: true, + params: params + ) + return layerSeq + } + + /// + /// Build a simple VisionTransformer model. + /// + /// - Parameters: + /// - size: The data input size. + /// - patch: Size of patch. + /// - nbLayers: Number of atttention branches. + /// - nbHeads: Number of head in attention branches. + /// - hiddenDim: Dimension of neurons in the main branch. + /// - mlpDim: Dimension of neurons in the MLP branch. + /// - mlpActivation: Activation function in the MLP branch. + /// - Returns: The model built. + /// + func _buildModel( + size: Int, + patch: Int, + nbLayers: Int, + nbHeads: Int, + hiddenDim: Int, + mlpDim: Int, + mlpActivation: String) -> Model + { + let context = ModelContext(name: "VisionTransformer", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: size, + height: size, + params: params + ) + + let extraClass: LayerSeq = Constant2Seq( + sequence: 1, nbNeurons: hiddenDim, params: params + ) + + var layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: patch, nbNeurons: hiddenDim, + activation: nil, biases: true, + params: params + ) + let sequence = layerSeq.sequence + 1 + + let posEmbedding: LayerSeq = Constant12Seq( + sequence: sequence, nbNeurons: hiddenDim, params: params + ) + + layerSeq = try! Concat1Seq( + layersPrev: [extraClass, layerSeq], params: params + ) + layerSeq = try! SumSeq( + layersPrev: [layerSeq, posEmbedding], params: params + ) + + for _ in 0.. label: 0) and half elements coming from + // cifar5 (dogs => label: 1). + cifar8.initSamples(batchSize: _batchSize / 2) + cifar5.initSamples(batchSize: _batchSize / 2) + + // Keep a subset of the dataset to have a quicker training. + cifar8.keep(500) + cifar5.keep(500) + + // Small trick to force full batches throughout the training: + // this enables us to set the ground truth once and for all. + let nbWholeBatches = + cifar8.nbSamples / cifar8.batchSize * cifar8.batchSize + cifar8.keep(nbWholeBatches) + cifar5.keep(nbWholeBatches) + + // Build a model with randomly initialized weights. + let transformer = _buildModel( + size: 32, + patch: 16, + nbLayers: 2, + nbHeads: 2, + hiddenDim: 16, + mlpDim: 32, + mlpActivation: GELU.str + ) + + // Initialize for training. + transformer.initialize(params: params, phase: .Training) + + let firstLayer: Input2D = transformer.layers.first as! Input2D + let lastLayer: MSE1D = transformer.layers.last as! MSE1D + + // Initialize the ground truth once and for all. + let groundTruth = MetalSharedBuffer(_batchSize, deviceID: 0) + let buffer = groundTruth.buffer + for elem in 0..<_batchSize / 2 + { + buffer[elem] = 0.0 + } + for elem in _batchSize / 2..<_batchSize + { + buffer[elem] = 1.0 + } + MetalKernel.get.upload([groundTruth]) + + let nbEpochs = 2 + for epoch in 0.. ratio1) } } diff --git a/Tests/GrAITests/Activation1DTests.swift b/Tests/GrAITests/Activation1DTests.swift index 3d1f4a55..67716e23 100644 --- a/Tests/GrAITests/Activation1DTests.swift +++ b/Tests/GrAITests/Activation1DTests.swift @@ -44,7 +44,7 @@ class Activation1DGradTests: Input1DMSE1DCase var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 5, activation: SoftReLU.str, biases: true, params: params @@ -53,7 +53,7 @@ class Activation1DGradTests: Input1DMSE1DCase switch model { case "FullyConnected": - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: activation, biases: true, params: params @@ -70,7 +70,7 @@ class Activation1DGradTests: Input1DMSE1DCase fatalError("Unreachable.") } - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: SoftReLU.str, biases: true, params: params @@ -164,6 +164,23 @@ class Activation1DGradTests: Input1DMSE1DCase run(trainer) } + func testFLGELUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "FullyConnected", activation: GELU.str + ) + run(trainer) + } + + func testFLGELUGPU() throws + { + let trainer = _buildTrainer( + model: "FullyConnected", activation: GELU.str + ) + run(trainer) + } + func testReLUCPU() throws { GrAI.Opti.CPU = true @@ -231,4 +248,21 @@ class Activation1DGradTests: Input1DMSE1DCase ) run(trainer) } + + func testGELUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Activation", activation: GELU.str + ) + run(trainer) + } + + func testGELUGPU() throws + { + let trainer = _buildTrainer( + model: "Activation", activation: GELU.str + ) + run(trainer) + } } diff --git a/Tests/GrAITests/Activation2DTests.swift b/Tests/GrAITests/Activation2DTests.swift index 03752df5..852e19f2 100644 --- a/Tests/GrAITests/Activation2DTests.swift +++ b/Tests/GrAITests/Activation2DTests.swift @@ -78,7 +78,7 @@ class Activation2DGradTests: Input2DMSE1DCase fatalError("Unreachable.") } - var head: Layer1D = FullyConnected( + var head: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: SoftReLU.str, biases: true, params: params ) @@ -256,6 +256,40 @@ class Activation2DGradTests: Input2DMSE1DCase run(trainer) } + func testConvGELUNoBNCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Convolution", activation: GELU.str, bn: false + ) + run(trainer) + } + + func testConvGELUBNCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Convolution", activation: GELU.str, bn: true + ) + run(trainer) + } + + func testConvGELUNoBNGPU() throws + { + let trainer = _buildTrainer( + model: "Convolution", activation: GELU.str, bn: false + ) + run(trainer) + } + + func testConvGELUBNGPU() throws + { + let trainer = _buildTrainer( + model: "Convolution", activation: GELU.str, bn: true + ) + run(trainer) + } + func testReLUCPU() throws { GrAI.Opti.CPU = true @@ -323,4 +357,21 @@ class Activation2DGradTests: Input2DMSE1DCase ) run(trainer) } + + func testGELUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Activation", activation: GELU.str, bn: false + ) + run(trainer) + } + + func testGELUGPU() throws + { + let trainer = _buildTrainer( + model: "Activation", activation: GELU.str, bn: false + ) + run(trainer) + } } diff --git a/Tests/GrAITests/ActivationSeqTests.swift b/Tests/GrAITests/ActivationSeqTests.swift new file mode 100644 index 00000000..5eda7487 --- /dev/null +++ b/Tests/GrAITests/ActivationSeqTests.swift @@ -0,0 +1,275 @@ +// +// ActivationSeqTests.swift +// GrAITests +// +// Created by Jean-François Reboud on 08/03/2023. +// + +import GrAIdient +import GrAITestsUtils + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class ActivationSeqGradTests: Input2DMSE1DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer(model: String, activation: String?) + -> GradTrainer + { + let trainer = GradTrainer( + name: "ActivationSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(model: model, activation: activation, context: context) + } + return trainer + } + + private func _buildModel( + model: String, + activation: String?, + context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + var layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + + switch model + { + case "FullyConnected": + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 5, + activation: activation, biases: true, + params: params + ) + + case "Activation": + layerSeq = ActivationSeq( + layerPrev: layerSeq, + activation: activation!, + params: params + ) + + default: + fatalError("Unreachable.") + } + + var head: Layer1D = AvgPoolSeq(layerPrev: layerSeq, params: params) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: SoftReLU.str, biases: true, params: params + ) + + _ = MSE1D(layerPrev: head, params: params) + } + + func testFLNoActivationCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "FullyConnected", activation: nil + ) + run(trainer) + } + + func testFLNoActivationGPU() throws + { + let trainer = _buildTrainer( + model: "FullyConnected", activation: nil + ) + run(trainer) + } + + func testFLReLUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "FullyConnected", activation: ReLU.str + ) + run(trainer) + } + + func testFLReLUGPU() throws + { + let trainer = _buildTrainer( + model: "FullyConnected", activation: ReLU.str + ) + run(trainer) + } + + func testFLLeakyReLUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "FullyConnected", activation: LeakyReLU.str + ) + run(trainer) + } + + func testFLLeakyReLUGPU() throws + { + let trainer = _buildTrainer( + model: "FullyConnected", activation: LeakyReLU.str + ) + run(trainer) + } + + func testFLSoftReLUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "FullyConnected", activation: SoftReLU.str + ) + run(trainer) + } + + func testFLSoftReLUGPU() throws + { + let trainer = _buildTrainer( + model: "FullyConnected", activation: SoftReLU.str + ) + run(trainer) + } + + func testFLSigmoidCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "FullyConnected", activation: Sigmoid.str + ) + run(trainer) + } + + func testFLSigmoidGPU() throws + { + let trainer = _buildTrainer( + model: "FullyConnected", activation: Sigmoid.str + ) + run(trainer) + } + + func testFLGELUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "FullyConnected", activation: GELU.str + ) + run(trainer) + } + + func testFLGELUGPU() throws + { + let trainer = _buildTrainer( + model: "FullyConnected", activation: GELU.str + ) + run(trainer) + } + + func testReLUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Activation", activation: ReLU.str + ) + run(trainer) + } + + func testReLUGPU() throws + { + let trainer = _buildTrainer( + model: "Activation", activation: ReLU.str + ) + run(trainer) + } + + func testLeakyReLUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Activation", activation: LeakyReLU.str + ) + run(trainer) + } + + func testLeakyReLUGPU() throws + { + let trainer = _buildTrainer( + model: "Activation", activation: LeakyReLU.str + ) + run(trainer) + } + + func testSoftReLUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Activation", activation: SoftReLU.str + ) + run(trainer) + } + + func testSoftReLUGPU() throws + { + let trainer = _buildTrainer( + model: "Activation", activation: SoftReLU.str + ) + run(trainer) + } + + func testSigmoidCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Activation", activation: Sigmoid.str + ) + run(trainer) + } + + func testSigmoidGPU() throws + { + let trainer = _buildTrainer( + model: "Activation", activation: Sigmoid.str + ) + run(trainer) + } + + func testGELUCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer( + model: "Activation", activation: GELU.str + ) + run(trainer) + } + + func testGELUGPU() throws + { + let trainer = _buildTrainer( + model: "Activation", activation: GELU.str + ) + run(trainer) + } +} diff --git a/Tests/GrAITests/Base/Input2DMSE1DCase.swift b/Tests/GrAITests/Base/IOCase.swift similarity index 76% rename from Tests/GrAITests/Base/Input2DMSE1DCase.swift rename to Tests/GrAITests/Base/IOCase.swift index 79b09cc3..11d147cd 100644 --- a/Tests/GrAITests/Base/Input2DMSE1DCase.swift +++ b/Tests/GrAITests/Base/IOCase.swift @@ -1,138 +1,85 @@ // -// Input2DMSE1DCase.swift +// IOCase.swift // GrAITests // -// Created by Jean-François Reboud on 15/10/2022. +// Created by Jean-François Reboud on 05/07/2023. // import XCTest import GrAIdient import GrAITestsUtils -/// -/// A class that will test a model with a structural hypothesis: -/// the model last layer is a MSE1D layer, the model first layer is a Input2D. -/// -class Input2DMSE1DCase: MSE1DCase +let NB_RETRY = 3 + +/// Use case with functions to get / set inputs and outputs. +protocol IOCase { - var height = 6 - var width = 6 + associatedtype DataT + associatedtype LossT - /// - /// A function to create/set data to the model. - /// - /// - Parameters: - /// - inputs: The data to set. - /// - model: The model. - /// - Returns: (The data, the batch size). - /// - func setData(_ inputs: [[Double]]?, _ model: Model) -> ([[Double]], Int) - { - let firstLayer = model.layers.first as! Input2D - let ins: [[Double]] - if let insTmp = inputs - { - ins = insTmp - } - else - { - ins = buildData(dim1: getBatchSize(model), dim2: height * width) - } - - if GrAI.Opti.GPU - { - try! firstLayer.setDataGPU(ins, format: .Neuron) - } - else - { - try! firstLayer.setDataCPU(ins, format: .Neuron) - } - return (ins, ins.count) - } + /// Batch size of data. + var batchSize: Int { get } + /// Optimizer parameters. + var optimizerParams: GrAI.Optimizer.Params { get } - /// - /// Copy a model. - /// - /// We must call the `initKernel` API. - /// - /// - Parameter model: The model. - /// - Returns: The transformed model. - /// - func copy(_ model: Model) -> Model - { - let modelNew = Model.copy(models: [model], inPlace: false)[0] - modelNew.initialize( - params: optimizerParams, - phase: .Inference, - deviceID: DEVICE_ID - ) - return modelNew - } + /// A list of functions that transform the model into another one. + var transforms: [(Model)->Model] { get } - /// - /// Copy a model in place. - /// - /// No need to call the `initKernel` API. - /// - /// - Parameter model: The model. - /// - Returns: The transformed model. - /// - func copyInPlace(_ model: Model) -> Model - { - let modelNew = Model.copy(models: [model], inPlace: true)[0] - modelNew.setupOptimizers(params: optimizerParams) - modelNew.phase = .Inference - return modelNew - } + /// A function to create/set data to the model. + func setData(_: DataT?, _: Model) -> (DataT, Int) + /// A function to get the loss of the model. + func getLoss(_: LossT, _: Model) -> Double + /// A function to create/set ground truth to the model. + func setLoss(_: LossT?, _: Model) -> LossT + /// A function that gets gradients of weights approximations. + func getGradientsApprox(_: LossT, _: Model) -> [Double] +} + +extension IOCase +{ /// - /// Resize a model. + /// Get the current batch size of data. /// - /// We must call the `initKernel` API. + /// This function allows to simulate the fact that the batch size of data may be smalling during the + /// last iteration of the training. /// /// - Parameter model: The model. - /// - Returns: The transformed model. + /// - Returns: The batch size of data. /// - func resize(_ model: Model) -> Model + func getBatchSize(_ model: Model) -> Int { - let modelsNew = Model.resize(models: [model], - imageWidth: 2 * width, - imageHeight: 2 * height, - inPlace: false) - let modelNew = Model.resize(models: modelsNew, - imageWidth: width, - imageHeight: height, - inPlace: false)[0] - modelNew.initialize( - params: optimizerParams, - phase: .Inference, - deviceID: DEVICE_ID - ) - return modelNew + if model.optimizerParams.step == model.optimizerParams.nbLoops-1 + { + return batchSize / 2 + } + else + { + return batchSize + } } /// - /// Resize a model in place. - /// - /// No need to call the `initKernel` API. + /// Create synthetic data. /// - /// - Parameter model: The model. - /// - Returns: The transformed model. + /// - Parameters: + /// - dim1: The first dimension of the data. + /// - dim2: The second dimension of the data. + /// - Returns: The created data. /// - func resizeInPlace(_ model: Model) -> Model + func buildData(dim1: Int, dim2: Int) -> [[T]] { - let modelsNew = Model.resize(models: [model], - imageWidth: 2 * width, - imageHeight: 2 * height, - inPlace: true) - let modelNew = Model.resize(models: modelsNew, - imageWidth: width, - imageHeight: height, - inPlace: true)[0] - modelNew.updateKernel(batchSize: batchSize) - modelNew.setupOptimizers(params: optimizerParams) - modelNew.phase = .Inference - return modelNew + var data = [[T]]() + for _ in 0.. Model + { + let modelNew = Model.copy(models: [model], inPlace: false)[0] + modelNew.initialize( + params: optimizerParams, + phase: .Inference, + deviceID: DEVICE_ID + ) + return modelNew + } + + /// + /// Copy a model in place. + /// + /// No need to call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func copyInPlace(_ model: Model) -> Model + { + let modelNew = Model.copy(models: [model], inPlace: true)[0] + modelNew.setupOptimizers(params: optimizerParams) + modelNew.phase = .Inference + return modelNew + } + + /// A list of functions that transform the model into another one. + var transforms: [(Model) -> Model] + { + get { + return [copy, copyInPlace] + } + } +} + +/// Use case where first layer is an Input2D. +protocol Input2DCase +{ + /// Height of the Input2D layer. + var height: Int { get } + /// Width of the Input2D layer. + var width: Int { get } + + /// Batch size of data. + var batchSize: Int { get } + /// Optimizer parameters. + var optimizerParams: GrAI.Optimizer.Params { get } +} + +extension Input2DCase +{ + /// + /// Copy a model. + /// + /// We must call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func copy(_ model: Model) -> Model + { + let modelNew = Model.copy(models: [model], inPlace: false)[0] + modelNew.initialize( + params: optimizerParams, + phase: .Inference, + deviceID: DEVICE_ID + ) + return modelNew + } + + /// + /// Copy a model in place. + /// + /// No need to call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func copyInPlace(_ model: Model) -> Model + { + let modelNew = Model.copy(models: [model], inPlace: true)[0] + modelNew.setupOptimizers(params: optimizerParams) + modelNew.phase = .Inference + return modelNew + } + + /// + /// Resize a model. + /// + /// We must call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func resize(_ model: Model) -> Model + { + let modelsNew = Model.resize(models: [model], + imageWidth: 2 * width, + imageHeight: 2 * height, + inPlace: false) + let modelNew = Model.resize(models: modelsNew, + imageWidth: width, + imageHeight: height, + inPlace: false)[0] + modelNew.initialize( + params: optimizerParams, + phase: .Inference, + deviceID: DEVICE_ID + ) + return modelNew + } + + /// + /// Resize a model in place. + /// + /// No need to call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func resizeInPlace(_ model: Model) -> Model + { + let modelsNew = Model.resize(models: [model], + imageWidth: 2 * width, + imageHeight: 2 * height, + inPlace: true) + let modelNew = Model.resize(models: modelsNew, + imageWidth: width, + imageHeight: height, + inPlace: true)[0] + modelNew.updateKernel(batchSize: batchSize) + modelNew.setupOptimizers(params: optimizerParams) + modelNew.phase = .Inference + return modelNew + } + + /// A list of functions that transform the model into another one. + var transforms: [(Model) -> Model] + { + get { + return [copy, copyInPlace, resize, resizeInPlace] + } + } +} diff --git a/Tests/GrAITests/Base/Input1D/Input1DBCE1DCase.swift b/Tests/GrAITests/Base/Input1D/Input1DBCE1DCase.swift new file mode 100644 index 00000000..04c4e82a --- /dev/null +++ b/Tests/GrAITests/Base/Input1D/Input1DBCE1DCase.swift @@ -0,0 +1,160 @@ +// +// Input1DBCE1DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 06/07/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a BCE1D layer, the model first layer is a Input1D. +/// +class Input1DBCE1DCase: XCTestCase, Input1DCase, IOCase +{ + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 3 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! BCE1D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + var values = [[Double]]() + let batchSize = getBatchSize(model) + for _ in 0.. Double + { + let lastLayer = model.layers.last as! BCE1D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + )) + } + else + { + return try! lastLayer.getLossCPU( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! BCE1D + return try! lastLayer.collectGradientsApprox( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + public func setData( + _ inputs: [[Float]]?, + _ model: Model) -> ([[Float]], Int) + { + let firstLayer = model.layers.first as! Input1D + let ins: [[Float]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: 1) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins, batchSize: ins.count, nbNeurons: 1 + ) + } + else + { + try! firstLayer.setDataCPU( + ins, batchSize: ins.count, nbNeurons: 1 + ) + } + return (ins, ins.count) + } +} diff --git a/Tests/GrAITests/Base/Input1D/Input1DBCESigmoid1DCase.swift b/Tests/GrAITests/Base/Input1D/Input1DBCESigmoid1DCase.swift new file mode 100644 index 00000000..b869b67a --- /dev/null +++ b/Tests/GrAITests/Base/Input1D/Input1DBCESigmoid1DCase.swift @@ -0,0 +1,160 @@ +// +// Input1DBCESigmoid1DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 07/07/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a BCESigmoid1D layer, the model first layer is a Input1D. +/// +class Input1DBCESigmoid1DCase: XCTestCase, Input1DCase, IOCase +{ + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 3 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! BCESigmoid1D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + var values = [[Double]]() + let batchSize = getBatchSize(model) + for _ in 0.. Double + { + let lastLayer = model.layers.last as! BCESigmoid1D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + )) + } + else + { + return try! lastLayer.getLossCPU( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! BCESigmoid1D + return try! lastLayer.collectGradientsApprox( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + public func setData( + _ inputs: [[Float]]?, + _ model: Model) -> ([[Float]], Int) + { + let firstLayer = model.layers.first as! Input1D + let ins: [[Float]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: 1) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins, batchSize: ins.count, nbNeurons: 1 + ) + } + else + { + try! firstLayer.setDataCPU( + ins, batchSize: ins.count, nbNeurons: 1 + ) + } + return (ins, ins.count) + } +} diff --git a/Tests/GrAITests/Base/Input1D/Input1DLinearError1DCase.swift b/Tests/GrAITests/Base/Input1D/Input1DLinearError1DCase.swift new file mode 100644 index 00000000..e6588e96 --- /dev/null +++ b/Tests/GrAITests/Base/Input1D/Input1DLinearError1DCase.swift @@ -0,0 +1,186 @@ +// +// Input1DLinearError1DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 10/10/2022. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a LinearError1D layer, the model first layer is a Input1D. +/// +class Input1DLinearError1DCase: XCTestCase, IOCase +{ + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 3 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! LinearError1D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + gt = buildData(dim1: getBatchSize(model), dim2: 1) + } + + if GrAI.Opti.GPU + { + try! lastLayer.lossDerivativeGPU() + } + else + { + try! lastLayer.lossDerivativeCPU() + } + return gt + } + + /// + /// A function to get loss of a model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The loss value. + /// + func getLoss(_ groundTruth: [[Double]], _ model: Model) -> Double + { + let lastLayer = model.layers.last as! LinearError1D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + )) + } + else + { + return try! lastLayer.getLossCPU( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! LinearError1D + return try! lastLayer.collectGradientsApprox( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData( + _ inputs: [[Float]]?, + _ model: Model + ) -> ([[Float]], Int) + { + let firstLayer = model.layers.first as! Input1D + let ins: [[Float]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: 1) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins, batchSize: ins.count, nbNeurons: 1 + ) + } + else + { + try! firstLayer.setDataCPU( + ins, batchSize: ins.count, nbNeurons: 1 + ) + } + return (ins, ins.count) + } + + /// + /// Copy a model and call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func copy(_ model: Model) -> Model + { + let modelNew = Model.copy(models: [model], inPlace: false)[0] + modelNew.initialize( + params: optimizerParams, + phase: .Inference, + deviceID: DEVICE_ID + ) + return modelNew + } + + /// + /// Copy a model in place: do not call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func copyInPlace(_ model: Model) -> Model + { + let modelNew = Model.copy(models: [model], inPlace: true)[0] + modelNew.setupOptimizers(params: optimizerParams) + modelNew.phase = .Inference + return modelNew + } + + /// A list of functions that transform the model into another one. + var transforms: [(Model) -> Model] + { + get { + return [copy, copyInPlace] + } + } +} diff --git a/Tests/GrAITests/Base/MSE1DCase.swift b/Tests/GrAITests/Base/Input1D/Input1DMSE1DCase.swift similarity index 56% rename from Tests/GrAITests/Base/MSE1DCase.swift rename to Tests/GrAITests/Base/Input1D/Input1DMSE1DCase.swift index bd6b099c..53b77e20 100644 --- a/Tests/GrAITests/Base/MSE1DCase.swift +++ b/Tests/GrAITests/Base/Input1D/Input1DMSE1DCase.swift @@ -1,8 +1,8 @@ // -// MSE1DCase.swift +// Input1DMSE1DCase.swift // GrAITests // -// Created by Jean-François Reboud on 10/10/2022. +// Created by Jean-François Reboud on 10/10/2022. // import XCTest @@ -11,12 +11,12 @@ import GrAITestsUtils /// /// A class that will test a model with a structural hypothesis: -/// the model last layer is a MSE1D layer. +/// the model last layer is a MSE1D layer, the model first layer is a Input1D. /// -class MSE1DCase: XCTestCase +class Input1DMSE1DCase: XCTestCase, Input1DCase, IOCase { /// Batch size of data. - var batchSize: Int! = nil + var batchSize: Int = -1 /// Optimizer parameters. var optimizerParams = GrAI.Optimizer.Params() @@ -31,50 +31,6 @@ class MSE1DCase: XCTestCase optimizerParams.nbLoops = 3 } - /// - /// Get the current batch size of data. - /// - /// This function allows to simulate the fact that the batch size of data may be smalling during the - /// last iteration of the training. - /// - /// - Parameter model: The model. - /// - Returns: The batch size of data. - /// - func getBatchSize(_ model: Model) -> Int - { - if model.optimizerParams.step == model.optimizerParams.nbLoops-1 - { - return batchSize / 2 - } - else - { - return batchSize - } - } - - /// - /// Create synthetic data. - /// - /// - Parameters: - /// - dim1: The first dimension of the data. - /// - dim2: The second dimension of the data. - /// - Returns: The created data. - /// - func buildData(dim1: Int, dim2: Int) -> [[T]] - { - var data = [[T]]() - for _ in 0.. [Double] { let lastLayer = model.layers.last as! MSE1D - return try! lastLayer.collectGradientsApprox(groundTruth) + return try! lastLayer.collectGradientsApprox( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + public func setData( + _ inputs: [[Float]]?, + _ model: Model) -> ([[Float]], Int) + { + let firstLayer = model.layers.first as! Input1D + let ins: [[Float]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: 1) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins, batchSize: ins.count, nbNeurons: 1 + ) + } + else + { + try! firstLayer.setDataCPU( + ins, batchSize: ins.count, nbNeurons: 1 + ) + } + return (ins, ins.count) } } diff --git a/Tests/GrAITests/Base/Input1DLinearError1DCase.swift b/Tests/GrAITests/Base/Input1DLinearError1DCase.swift deleted file mode 100644 index 9d964feb..00000000 --- a/Tests/GrAITests/Base/Input1DLinearError1DCase.swift +++ /dev/null @@ -1,410 +0,0 @@ -// -// Input1DLinearError1DCase.swift -// GrAITests -// -// Created by Jean-François Reboud on 10/10/2022. -// - -import XCTest -import GrAIdient -import GrAITestsUtils - -/// -/// A class that will test a model with a structural hypothesis: -/// the model last layer is a LinearError1D layer, the model first layer is a Input1D. -/// -class Input1DLinearError1DCase: LinearError1DCase -{ - /// - /// A function to create/set data to the model. - /// - /// - Parameters: - /// - inputs: The data to set. - /// - model: The model. - /// - Returns: (The data, the batch size). - /// - func setData( - _ inputs: [[Float]]?, - _ model: Model - ) -> ([[Float]], Int) - { - let firstLayer = model.layers.first as! Input1D - let ins: [[Float]] - if let insTmp = inputs - { - ins = insTmp - } - else - { - ins = buildData(dim1: getBatchSize(model), dim2: 1) - } - - if GrAI.Opti.GPU - { - try! firstLayer.setDataGPU(ins) - } - else - { - try! firstLayer.setDataCPU(ins) - } - return (ins, ins.count) - } - - /// - /// Copy a model and call the `initKernel` API. - /// - /// - Parameter model: The model. - /// - Returns: The transformed model. - /// - func copy(_ model: Model) -> Model - { - let modelNew = Model.copy(models: [model], inPlace: false)[0] - modelNew.initialize( - params: optimizerParams, - phase: .Inference, - deviceID: DEVICE_ID - ) - return modelNew - } - - /// - /// Copy a model in place: do not call the `initKernel` API. - /// - /// - Parameter model: The model. - /// - Returns: The transformed model. - /// - func copyInPlace(_ model: Model) -> Model - { - let modelNew = Model.copy(models: [model], inPlace: true)[0] - modelNew.setupOptimizers(params: optimizerParams) - modelNew.phase = .Inference - return modelNew - } - - /// - /// Run Gradient Checking test. - /// - /// The goal is to compare the gradients of weights that are computed through `backward` - /// to an estimation that is being computed through `forwardGC`. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: GradTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.000001) - { - let model = trainer.model! - let lastLayer = model.layers.last as! LinearError1D - let layersGraph = model.getGraph(lastLayer) - - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - layersGraph: layersGraph, - setData: self.setData, - setLoss: self.setLoss, - getGradientsApprox: self.getGradientsApprox) - { - (gradDiff: Double) in - if gradDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Flow test. - /// - /// The goal is to compare the gradients of weights computed in the CPU execution context with - /// the gradients of weights computed in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: FlowTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.000001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss) - { - (gradDiff: Double) in - if gradDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Flow Reset test. - /// - /// The goal is to compare the gradients of weights computed in the CPU execution context with - /// the gradients of weights computed in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: FlowResetTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.000001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss) - { - (gradDiff: Double) in - if gradDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Flow Reverse test. - /// - /// The goal is to compare the gradients of weights computed in the CPU execution context with - /// the gradients of weights computed in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: FlowReverseTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.000001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss) - { - (gradDiff: Double) in - if gradDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Inference test. - /// - /// The goal is to compare the losses computed in the CPU execution context with - /// the losses computed in the GPU execution context during the inference phase. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: InferenceTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss, - getLoss: self.getLoss) - { - (lossDiff: Double) in - if lossDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Loading test. - /// - /// The goal is to compare the losses computed in the CPU execution after havinng loaded the - /// model from the disk and do the same in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: LoadTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss, - getLoss: self.getLoss) - { - (diffCPU: Double, diffGPU: Double) in - if diffCPU > diffThreshold - { - throw TestError.Numeric - } - if diffGPU > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Transform tests. - /// - /// The goal is to compare the losses computed in the CPU execution - /// after transforming the model and do the same in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: TransformTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - transforms: [self.copy, self.copyInPlace], - setData: self.setData, - setLoss: self.setLoss, - getLoss: self.getLoss) - { - (diffCPU: Double, diffGPU: Double) in - if diffCPU > diffThreshold - { - throw TestError.Numeric - } - if diffGPU > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Clipping test. - /// - /// The goal is to compare the norm of the gradients of the weights with a threshold. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - normClipping: The threshold above which gradients must be cut. - /// - func run( - _ trainer: NormTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.001, - normClipping: Double = 0.001) - { - optimizerParams.gradientClipping = true - optimizerParams.normThreshold = normClipping - trainer.optimizerParams = optimizerParams - - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss) - { - (normDiff: Double) throws in - if normDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } -} diff --git a/Tests/GrAITests/Base/Input1DMSE1DCase.swift b/Tests/GrAITests/Base/Input1DMSE1DCase.swift deleted file mode 100644 index 13aa9984..00000000 --- a/Tests/GrAITests/Base/Input1DMSE1DCase.swift +++ /dev/null @@ -1,415 +0,0 @@ -// -// Input1DMSE1DCase.swift -// GrAITests -// -// Created by Jean-François Reboud on 10/10/2022. -// - -import XCTest -import GrAIdient -import GrAITestsUtils - -let NB_RETRY = 3 - -/// -/// A class that will test a model with a structural hypothesis: -/// the model last layer is a MSE1D layer, the model first layer is a Input1D. -/// -class Input1DMSE1DCase: MSE1DCase -{ - /// - /// A function to create/set data to the model. - /// - /// - Parameters: - /// - inputs: The data to set. - /// - model: The model. - /// - Returns: (The data, the batch size). - /// - public func setData( - _ inputs: [[Float]]?, - _ model: Model) -> ([[Float]], Int) - { - let firstLayer = model.layers.first as! Input1D - let ins: [[Float]] - if let insTmp = inputs - { - ins = insTmp - } - else - { - ins = buildData(dim1: getBatchSize(model), dim2: 1) - } - - if GrAI.Opti.GPU - { - try! firstLayer.setDataGPU(ins) - } - else - { - try! firstLayer.setDataCPU(ins) - } - return (ins, ins.count) - } - - /// - /// Copy a model. - /// - /// We must call the `initKernel` API. - /// - /// - Parameter model: The model. - /// - Returns: The transformed model. - /// - func copy(_ model: Model) -> Model - { - let modelNew = Model.copy(models: [model], inPlace: false)[0] - modelNew.initialize( - params: optimizerParams, - phase: .Inference, - deviceID: DEVICE_ID - ) - return modelNew - } - - /// - /// Copy a model in place. - /// - /// No need to call the `initKernel` API. - /// - /// - Parameter model: The model. - /// - Returns: The transformed model. - /// - func copyInPlace(_ model: Model) -> Model - { - let modelNew = Model.copy(models: [model], inPlace: true)[0] - modelNew.setupOptimizers(params: optimizerParams) - modelNew.phase = .Inference - return modelNew - } - - /// - /// Run Gradient Checking test. - /// - /// The goal is to compare the gradients of weights that are computed through `backward` - /// to an estimation that is being computed through `forwardGC`. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: GradTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.000001) - { - let model = trainer.model! - let lastLayer = model.layers.last as! MSE1D - let layersGraph = model.getGraph(lastLayer) - - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - layersGraph: layersGraph, - setData: self.setData, - setLoss: self.setLoss, - getGradientsApprox: self.getGradientsApprox) - { - (gradDiff: Double) in - if gradDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Flow test. - /// - /// The goal is to compare the gradients of weights computed in the CPU execution context with - /// the gradients of weights computed in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: FlowTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.000001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss) - { - (gradDiff: Double) in - if gradDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Flow Reset test. - /// - /// The goal is to compare the gradients of weights computed in the CPU execution context with - /// the gradients of weights computed in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: FlowResetTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.000001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss) - { - (gradDiff: Double) in - if gradDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Flow Reverse test. - /// - /// The goal is to compare the gradients of weights computed in the CPU execution context with - /// the gradients of weights computed in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: FlowReverseTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.000001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss) - { - (gradDiff: Double) in - if gradDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Inference test. - /// - /// The goal is to compare the losses computed in the CPU execution context with - /// the losses computed in the GPU execution context during the inference phase. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: InferenceTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss, - getLoss: self.getLoss) - { - (lossDiff: Double) in - if lossDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Loading test. - /// - /// The goal is to compare the losses computed in the CPU execution after havinng loaded the - /// model from the disk and do the same in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: LoadTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss, - getLoss: self.getLoss) - { - (diffCPU: Double, diffGPU: Double) in - if diffCPU > diffThreshold - { - throw TestError.Numeric - } - if diffGPU > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Transform tests. - /// - /// The goal is to compare the losses computed in the CPU execution - /// after transforming the model and do the same in the GPU execution context. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - func run( - _ trainer: TransformTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.001) - { - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - transforms: [self.copy, self.copyInPlace], - setData: self.setData, - setLoss: self.setLoss, - getLoss: self.getLoss) - { - (diffCPU: Double, diffGPU: Double) in - if diffCPU > diffThreshold - { - throw TestError.Numeric - } - if diffGPU > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } - - /// - /// Run Clipping test. - /// - /// The goal is to compare the norm of the gradients of the weights with a threshold. - /// - /// - Parameters: - /// - trainer: The testing pipeline to run. - /// - nbRetry: The maximum number we can retry the test. - /// - diffThreshold: The threshold above which the relative difference is too high. - /// - normClipping: The threshold above which gradients must be cut. - /// - func run( - _ trainer: NormTrainer, - nbRetry: Int = NB_RETRY, - diffThreshold: Double = 0.001, - normClipping: Double = 0.001) - { - optimizerParams.gradientClipping = true - optimizerParams.normThreshold = normClipping - trainer.optimizerParams = optimizerParams - - retryNumeric( - nbRetry: nbRetry, - { - () throws in - try trainer.run( - setData: self.setData, - setLoss: self.setLoss) - { - (normDiff: Double) throws in - if normDiff > diffThreshold - { - throw TestError.Numeric - } - } - }, - { - () in - XCTAssert(false) - } - ) - } -} diff --git a/Tests/GrAITests/Base/FTFrequences2DMSE1DCase.swift b/Tests/GrAITests/Base/Input2D/FTFrequences2DMSE1DCase.swift similarity index 100% rename from Tests/GrAITests/Base/FTFrequences2DMSE1DCase.swift rename to Tests/GrAITests/Base/Input2D/FTFrequences2DMSE1DCase.swift diff --git a/Tests/GrAITests/Base/Input2D/Input2DBCE2DCase.swift b/Tests/GrAITests/Base/Input2D/Input2DBCE2DCase.swift new file mode 100644 index 00000000..fad660b5 --- /dev/null +++ b/Tests/GrAITests/Base/Input2D/Input2DBCE2DCase.swift @@ -0,0 +1,184 @@ +// +// Input2DBCE2DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 06/07/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a BCE2D layer, the model first layer is a Input2D. +/// +class Input2DBCE2DCase: XCTestCase, Input2DCase, IOCase +{ + /// Height of the Input2D layer. + var height = 6 + /// Width of the Input2D layer. + var width = 6 + + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 2 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! BCE2D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + var values = [[Double]]() + for _ in 0.. Double + { + let lastLayer = model.layers.last as! BCE2D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + )) + } + else + { + return try! lastLayer.getLossCPU( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! BCE2D + return try! lastLayer.collectGradientsApprox( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData(_ inputs: [[Double]]?, _ model: Model) -> ([[Double]], Int) + { + let firstLayer = model.layers.first as! Input2D + let ins: [[Double]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! firstLayer.setDataCPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return (ins, ins.count) + } +} diff --git a/Tests/GrAITests/Base/Input2D/Input2DBCESigmoid2DCase.swift b/Tests/GrAITests/Base/Input2D/Input2DBCESigmoid2DCase.swift new file mode 100644 index 00000000..69196dcc --- /dev/null +++ b/Tests/GrAITests/Base/Input2D/Input2DBCESigmoid2DCase.swift @@ -0,0 +1,184 @@ +// +// Input2DBCESigmoid2DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 07/07/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a BCESigmoid2D layer, the model first layer is a Input2D. +/// +class Input2DBCESigmoid2DCase: XCTestCase, Input2DCase, IOCase +{ + /// Height of the Input2D layer. + var height = 6 + /// Width of the Input2D layer. + var width = 6 + + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 2 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! BCESigmoid2D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + var values = [[Double]]() + for _ in 0.. Double + { + let lastLayer = model.layers.last as! BCESigmoid2D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + )) + } + else + { + return try! lastLayer.getLossCPU( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! BCESigmoid2D + return try! lastLayer.collectGradientsApprox( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData(_ inputs: [[Double]]?, _ model: Model) -> ([[Double]], Int) + { + let firstLayer = model.layers.first as! Input2D + let ins: [[Double]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! firstLayer.setDataCPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return (ins, ins.count) + } +} diff --git a/Tests/GrAITests/Base/Input2D/Input2DMSE1DCase.swift b/Tests/GrAITests/Base/Input2D/Input2DMSE1DCase.swift new file mode 100644 index 00000000..6c4bd08b --- /dev/null +++ b/Tests/GrAITests/Base/Input2D/Input2DMSE1DCase.swift @@ -0,0 +1,250 @@ +// +// Input2DMSE1DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 15/10/2022. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a MSE1D layer, the model first layer is a Input2D. +/// +class Input2DMSE1DCase: XCTestCase, IOCase +{ + var height = 6 + var width = 6 + + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 3 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! MSE1D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + gt = buildData(dim1: getBatchSize(model), dim2: 1) + } + + if GrAI.Opti.GPU + { + try! lastLayer.lossDerivativeGPU( + gt, batchSize: gt.count, nbNeurons: 1 + ) + } + else + { + try! lastLayer.lossDerivativeCPU( + gt, batchSize: gt.count, nbNeurons: 1 + ) + } + return gt + } + + /// + /// A function to get loss of a model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The loss value. + /// + func getLoss(_ groundTruth: [[Double]], _ model: Model) -> Double + { + let lastLayer = model.layers.last as! MSE1D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + )) + } + else + { + return try! lastLayer.getLossCPU( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! MSE1D + return try! lastLayer.collectGradientsApprox( + groundTruth, batchSize: groundTruth.count, nbNeurons: 1 + ) + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData(_ inputs: [[Double]]?, _ model: Model) -> ([[Double]], Int) + { + let firstLayer = model.layers.first as! Input2D + let ins: [[Double]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! firstLayer.setDataCPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return (ins, ins.count) + } + + /// + /// Copy a model. + /// + /// We must call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func copy(_ model: Model) -> Model + { + let modelNew = Model.copy(models: [model], inPlace: false)[0] + modelNew.initialize( + params: optimizerParams, + phase: .Inference, + deviceID: DEVICE_ID + ) + return modelNew + } + + /// + /// Copy a model in place. + /// + /// No need to call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func copyInPlace(_ model: Model) -> Model + { + let modelNew = Model.copy(models: [model], inPlace: true)[0] + modelNew.setupOptimizers(params: optimizerParams) + modelNew.phase = .Inference + return modelNew + } + + /// + /// Resize a model. + /// + /// We must call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func resize(_ model: Model) -> Model + { + let modelsNew = Model.resize(models: [model], + imageWidth: 2 * width, + imageHeight: 2 * height, + inPlace: false) + let modelNew = Model.resize(models: modelsNew, + imageWidth: width, + imageHeight: height, + inPlace: false)[0] + modelNew.initialize( + params: optimizerParams, + phase: .Inference, + deviceID: DEVICE_ID + ) + return modelNew + } + + /// + /// Resize a model in place. + /// + /// No need to call the `initKernel` API. + /// + /// - Parameter model: The model. + /// - Returns: The transformed model. + /// + func resizeInPlace(_ model: Model) -> Model + { + let modelsNew = Model.resize(models: [model], + imageWidth: 2 * width, + imageHeight: 2 * height, + inPlace: true) + let modelNew = Model.resize(models: modelsNew, + imageWidth: width, + imageHeight: height, + inPlace: true)[0] + modelNew.updateKernel(batchSize: batchSize) + modelNew.setupOptimizers(params: optimizerParams) + modelNew.phase = .Inference + return modelNew + } + + /// A list of functions that transform the model into another one. + var transforms: [(Model) -> Model] + { + get { + return [copy, copyInPlace, resize, resizeInPlace] + } + } +} diff --git a/Tests/GrAITests/Base/Input2D/Input2DMSE2DCase.swift b/Tests/GrAITests/Base/Input2D/Input2DMSE2DCase.swift new file mode 100644 index 00000000..4cf3b5e3 --- /dev/null +++ b/Tests/GrAITests/Base/Input2D/Input2DMSE2DCase.swift @@ -0,0 +1,174 @@ +// +// Input2DMSE2DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 05/03/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a MSE2D layer, the model first layer is a Input2D. +/// +class Input2DMSE2DCase: XCTestCase, Input2DCase, IOCase +{ + /// Height of the Input2D layer. + var height = 6 + /// Width of the Input2D layer. + var width = 6 + + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 2 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! MSE2D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + gt = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! lastLayer.lossDerivativeGPU( + gt.reduce([], +), + batchSize: gt.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! lastLayer.lossDerivativeCPU( + gt.reduce([], +), + batchSize: gt.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return gt + } + + /// + /// A function to get loss of a model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The loss value. + /// + func getLoss(_ groundTruth: [[Double]], _ model: Model) -> Double + { + let lastLayer = model.layers.last as! MSE2D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + )) + } + else + { + return try! lastLayer.getLossCPU( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! MSE2D + return try! lastLayer.collectGradientsApprox( + groundTruth.reduce([], +), + batchSize: groundTruth.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData(_ inputs: [[Double]]?, _ model: Model) -> ([[Double]], Int) + { + let firstLayer = model.layers.first as! Input2D + let ins: [[Double]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! firstLayer.setDataCPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return (ins, ins.count) + } +} diff --git a/Tests/GrAITests/Base/Input2D/Input2DSimilarityBatchError2DCase.swift b/Tests/GrAITests/Base/Input2D/Input2DSimilarityBatchError2DCase.swift new file mode 100644 index 00000000..01ab5196 --- /dev/null +++ b/Tests/GrAITests/Base/Input2D/Input2DSimilarityBatchError2DCase.swift @@ -0,0 +1,152 @@ +// +// Input2DSimilarityBatchError2DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 14/05/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a SimilarityBatchError2D layer, the model first layer is a Input2D. +/// +class Input2DSimilarityBatchError2DCase: XCTestCase, Input2DCase, IOCase +{ + /// Height of the Input2D layer. + var height = 6 + /// Width of the Input2D layer. + var width = 6 + + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 2 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! SimilarityBatchError2D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + gt = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! lastLayer.lossDerivativeGPU() + } + else + { + lastLayer.lossDerivativeCPU() + } + return gt + } + + /// + /// A function to get loss of a model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The loss value. + /// + func getLoss(_ groundTruth: [[Double]], _ model: Model) -> Double + { + let lastLayer = model.layers.last as! SimilarityBatchError2D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU()) + } + else + { + return lastLayer.getLossCPU() + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! SimilarityBatchError2D + return lastLayer.collectGradientsApprox() + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData( + _ inputs: [[Float]]?, + _ model: Model + ) -> ([[Float]], Int) + { + let firstLayer = model.layers.first as! Input2D + let ins: [[Float]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! firstLayer.setDataCPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return (ins, ins.count) + } +} diff --git a/Tests/GrAITests/Base/Input2D/Input2DSimilarityError2DCase.swift b/Tests/GrAITests/Base/Input2D/Input2DSimilarityError2DCase.swift new file mode 100644 index 00000000..fb9fb282 --- /dev/null +++ b/Tests/GrAITests/Base/Input2D/Input2DSimilarityError2DCase.swift @@ -0,0 +1,152 @@ +// +// Input2DSimilarityError2DCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 29/05/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a SimilarityError2D layer, the model first layer is a Input2D. +/// +class Input2DSimilarityError2DCase: XCTestCase, Input2DCase, IOCase +{ + /// Height of the Input2D layer. + var height = 6 + /// Width of the Input2D layer. + var width = 6 + + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 2 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! SimilarityError2D + let gt: [[Double]] + if let groundTruthTmp = groundTruth + { + gt = groundTruthTmp + } + else + { + gt = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! lastLayer.lossDerivativeGPU() + } + else + { + lastLayer.lossDerivativeCPU() + } + return gt + } + + /// + /// A function to get loss of a model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The loss value. + /// + func getLoss(_ groundTruth: [[Double]], _ model: Model) -> Double + { + let lastLayer = model.layers.last as! SimilarityError2D + if GrAI.Opti.GPU + { + return Double(try! lastLayer.getLossGPU()) + } + else + { + return lastLayer.getLossCPU() + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + let lastLayer = model.layers.last as! SimilarityError2D + return lastLayer.collectGradientsApprox() + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData( + _ inputs: [[Float]]?, + _ model: Model + ) -> ([[Float]], Int) + { + let firstLayer = model.layers.first as! Input2D + let ins: [[Float]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! firstLayer.setDataCPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return (ins, ins.count) + } +} diff --git a/Tests/GrAITests/Base/LinearError1DCase.swift b/Tests/GrAITests/Base/Input2D/Input2DVQ2DCase.swift similarity index 51% rename from Tests/GrAITests/Base/LinearError1DCase.swift rename to Tests/GrAITests/Base/Input2D/Input2DVQ2DCase.swift index 531baf3d..d39c8496 100644 --- a/Tests/GrAITests/Base/LinearError1DCase.swift +++ b/Tests/GrAITests/Base/Input2D/Input2DVQ2DCase.swift @@ -1,8 +1,8 @@ // -// LinearError1DCase.swift +// Input2DVQ2DCase.swift // GrAITests // -// Created by Jean-François Reboud on 10/10/2022. +// Created by Jean-François Reboud on 06/07/2023. // import XCTest @@ -11,12 +11,17 @@ import GrAITestsUtils /// /// A class that will test a model with a structural hypothesis: -/// the model last layer is a LinearError1D layer. -/// -class LinearError1DCase: XCTestCase +/// the model last layer is a VQ2D layer, the model first layer is a Input2D. +/// +class Input2DVQ2DCase: XCTestCase, Input2DCase, IOCase { + /// Height of the Input2D layer. + var height = 6 + /// Width of the Input2D layer. + var width = 6 + /// Batch size of data. - var batchSize: Int! = nil + var batchSize: Int = -1 /// Optimizer parameters. var optimizerParams = GrAI.Optimizer.Params() @@ -28,51 +33,7 @@ class LinearError1DCase: XCTestCase GrAI.Opti.GPU = true setOptimizerParams(params: &optimizerParams) - optimizerParams.nbLoops = 3 - } - - /// - /// Get the current batch size of data. - /// - /// This function allows to simulate the fact that the batch size of data may be smalling during the - /// last iteration of the training. - /// - /// - Parameter model: The model. - /// - Returns: The batch size of data. - /// - func getBatchSize(_ model: Model) -> Int - { - if model.optimizerParams.step == model.optimizerParams.nbLoops-1 - { - return batchSize / 2 - } - else - { - return batchSize - } - } - - /// - /// Create synthetic data. - /// - /// - Parameters: - /// - dim1: The first dimension of the data. - /// - dim2: The second dimension of the data. - /// - Returns: The created data. - /// - func buildData(dim1: Int, dim2: Int) -> [[T]] - { - var data = [[T]]() - for _ in 0.. [[Double]] { - let lastLayer = model.layers.last as! LinearError1D - let gt: [[Double]] - if let groundTruthTmp = groundTruth - { - gt = groundTruthTmp - } - else - { - gt = buildData(dim1: getBatchSize(model), dim2: 1) - } - + let lastLayer = model.layers.last as! VQ2D if GrAI.Opti.GPU { try! lastLayer.lossDerivativeGPU() @@ -104,7 +55,7 @@ class LinearError1DCase: XCTestCase { try! lastLayer.lossDerivativeCPU() } - return gt + return [[Double]]() } /// @@ -117,14 +68,14 @@ class LinearError1DCase: XCTestCase /// func getLoss(_ groundTruth: [[Double]], _ model: Model) -> Double { - let lastLayer = model.layers.last as! LinearError1D + let lastLayer = model.layers.last as! VQ2D if GrAI.Opti.GPU { - return Double(try! lastLayer.getLossGPU(groundTruth)) + return try! lastLayer.getLossGPU() } else { - return try! lastLayer.getLossCPU(groundTruth) + return lastLayer.getLossCPU() } } @@ -140,7 +91,48 @@ class LinearError1DCase: XCTestCase _ groundTruth: [[Double]], _ model: Model) -> [Double] { - let lastLayer = model.layers.last as! LinearError1D - return try! lastLayer.collectGradientsApprox(groundTruth) + fatalError("Not implemented.") + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData(_ inputs: [[Double]]?, _ model: Model) -> ([[Double]], Int) + { + let firstLayer = model.layers.first as! Input2D + let ins: [[Double]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! firstLayer.setDataCPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return (ins, ins.count) } } diff --git a/Tests/GrAITests/Base/Input2D/Input2DVQSeqCase.swift b/Tests/GrAITests/Base/Input2D/Input2DVQSeqCase.swift new file mode 100644 index 00000000..7a308997 --- /dev/null +++ b/Tests/GrAITests/Base/Input2D/Input2DVQSeqCase.swift @@ -0,0 +1,138 @@ +// +// Input2DVQSeqCase.swift +// GrAITests +// +// Created by Jean-François Reboud on 06/07/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +/// +/// A class that will test a model with a structural hypothesis: +/// the model last layer is a VQSeq layer, the model first layer is a Input2D. +/// +class Input2DVQSeqCase: XCTestCase, Input2DCase, IOCase +{ + /// Height of the Input2D layer. + var height = 6 + /// Width of the Input2D layer. + var width = 6 + + /// Batch size of data. + var batchSize: Int = -1 + /// Optimizer parameters. + var optimizerParams = GrAI.Optimizer.Params() + + /// Systematic call before test begins. + override func setUp() + { + batchSize = 5 + _ = MetalKernel.get + GrAI.Opti.GPU = true + + setOptimizerParams(params: &optimizerParams) + optimizerParams.nbLoops = 2 + } + + /// + /// A function to create/set ground truth to the model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The ground truth. + /// + func setLoss(_ groundTruth: [[Double]]?, _ model: Model) -> [[Double]] + { + let lastLayer = model.layers.last as! VQSeq + if GrAI.Opti.GPU + { + try! lastLayer.lossDerivativeGPU() + } + else + { + try! lastLayer.lossDerivativeCPU() + } + return [[Double]]() + } + + /// + /// A function to get loss of a model. + /// + /// - Parameters: + /// - groundTruth: The ground truth to set. + /// - model: The model. + /// - Returns: The loss value. + /// + func getLoss(_ groundTruth: [[Double]], _ model: Model) -> Double + { + let lastLayer = model.layers.last as! VQSeq + if GrAI.Opti.GPU + { + return try! lastLayer.getLossGPU() + } + else + { + return lastLayer.getLossCPU() + } + } + + /// + /// A function to get the gradients of weights approximations.. + /// + /// - Parameters: + /// - groundTruth: The ground truth. + /// - model: The model. + /// - Returns: The gradients of weights approximations. + /// + func getGradientsApprox( + _ groundTruth: [[Double]], + _ model: Model) -> [Double] + { + fatalError("Not implemented.") + } + + /// + /// A function to create/set data to the model. + /// + /// - Parameters: + /// - inputs: The data to set. + /// - model: The model. + /// - Returns: (The data, the batch size). + /// + func setData(_ inputs: [[Double]]?, _ model: Model) -> ([[Double]], Int) + { + let firstLayer = model.layers.first as! Input2D + let ins: [[Double]] + if let insTmp = inputs + { + ins = insTmp + } + else + { + ins = buildData(dim1: getBatchSize(model), dim2: height * width) + } + + if GrAI.Opti.GPU + { + try! firstLayer.setDataGPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + else + { + try! firstLayer.setDataCPU( + ins.reduce([], +), + batchSize: ins.count, + nbChannels: 1, height: height, width: width, + format: .Neuron + ) + } + return (ins, ins.count) + } +} diff --git a/Tests/GrAITests/ClippingTests.swift b/Tests/GrAITests/ClippingTests.swift index 65d6951e..06faf51e 100644 --- a/Tests/GrAITests/ClippingTests.swift +++ b/Tests/GrAITests/ClippingTests.swift @@ -34,19 +34,19 @@ class ClippingTests: Input1DMSE1DCase var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: LeakyReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: LeakyReLU.str, biases: true, params: params diff --git a/Tests/GrAITests/ImageTests.swift b/Tests/GrAITests/ImageTests.swift new file mode 100644 index 00000000..8221dec4 --- /dev/null +++ b/Tests/GrAITests/ImageTests.swift @@ -0,0 +1,613 @@ +// +// ImageTests.swift +// GrAITests +// +// Created by Jean-François Reboud on 19/05/2023. +// + +import Foundation +import XCTest +import GrAIdient + +/// Test operations on images. +@available(macOS 13.0, *) +class ImageTests: XCTestCase +{ + /// Directory containing input images. + let _inputURL = URL(string: #file)! + .deletingLastPathComponent() + .deletingLastPathComponent() + .appending(path: "data") + .appending(path: "in") + .appending(path: "224x224") + /// Directory containing rerence images. + let _referenceURL = URL(string: #file)! + .deletingLastPathComponent() + .deletingLastPathComponent() + .appending(path: "data") + .appending(path: "out") + .appending(path: "augmentation") + + /// Input images. + let _imageNames = [ + "harp", + "monastery", + "snail" + ] + + /// Directory to dump outputs from the tests. + let _outputDir = NSTemporaryDirectory() + + /// Size of one image (height and width are the same). + let _size = 224 + + var imagesURL: [URL] + { + get { + return [URL](repeating: _inputURL, count: 3).enumerated().map + { + (index, element) in + return element.appending(path: "\(_imageNames[index]).png") + } + } + } + + override func setUp() + { + _ = MetalKernel.get + GrAI.Opti.GPU = true + } + + private func _buildModel( + modelName: String, + parameters: Any) -> (Model, Input2D, Layer2D) + { + let context = ModelContext(name: "Image", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 3, width: _size, height: _size, params: params + ) + + switch modelName + { + case "Rotate": + let p = parameters as! Double + layer = try! Rotate2D( + layerPrev: layer, + anglesList: [p], + padValue: 0.0, + params: params + ) + + case "FlipHorizontal": + let p = parameters as! Double + layer = FlipHorizontal2D( + layerPrev: layer, + probability: p, + params: params + ) + + case "FlipVertical": + let p = parameters as! Double + layer = FlipVertical2D( + layerPrev: layer, + probability: p, + params: params + ) + + case "ColorJitterHSV": + let p = parameters as! (Range, Range, Range) + layer = try! ColorJitterHSV( + layerPrev: layer, + rangeH: p.0, + rangeS: p.1, + rangeV: p.2, + params: params + ) + + default: + fatalError("Unreachable.") + } + + let model = Model(model: context.model, modelsPrev: []) + model.initKernel() + + let firstLayer = model.layers.first as! Input2D + let lastLayer = model.layers.last as! Layer2D + + return (model, firstLayer, lastLayer) + } + + private func _compareCPU(lastLayer: Layer2D, suffix: String) + { + for (elem, name) in _imageNames.enumerated() + { + let pixelsOut: [Float] = lastLayer.getOutsCPU(elem: elem) + var pixels = Image.toPixel([pixelsOut]) + pixels = Image.toRGB(pixels, width: _size, height: _size) + + let image = Image.buildImage( + pixels: pixels[0], + width: _size, height: _size + ) + try! image.save( + url: URL(fileURLWithPath: _outputDir) + .appending(path: "\(name)_\(suffix).png") + ) + + let imageRef = NSImage( + byReferencingFile: _referenceURL + .appending(path: "\(name)_\(suffix).png").path + )! + let pixelsRef = try! imageRef.extractPaddedPixels( + width: CGFloat(_size), height: CGFloat(_size) + ) + XCTAssert(pixels[0] == pixelsRef) + } + } + + private func _compareGPU(lastLayer: Layer2D, suffix: String) + { + let pixelsBatch = Image.extractPixels( + lastLayer.outs, + width: _size, height: _size + ) + for (elem, pixels) in pixelsBatch.enumerated() + { + let image = Image.buildImage( + pixels: pixels, + width: _size, height: _size + ) + try! image.save( + url: URL(fileURLWithPath: _outputDir) + .appending(path: "\(_imageNames[elem])_\(suffix).png") + ) + + let imageRef = NSImage( + byReferencingFile: _referenceURL + .appending(path: "\(_imageNames[elem])_\(suffix).png").path + )! + let pixelsRef = try! imageRef.extractPaddedPixels( + width: CGFloat(_size), height: CGFloat(_size) + ) + + let test1 = pixelsBatch[elem] == pixelsRef + if !test1 + { + var nbFail = 0 + for (val1, val2) in zip(pixelsBatch[elem], pixelsRef) + { + if val1 != val2 + { + nbFail += 1 + } + } + let ratioFail = Double(nbFail) / Double(pixelsRef.count) * 100.0 + let test2 = ratioFail < 0.1 + + if !test2 + { + for (val1, val2) in zip(pixelsBatch[elem], pixelsRef) + { + if val1 != val2 + { + let diff = abs(Double(val1) - Double(val2)) + let test3 = diff <= 1 + XCTAssert(test3) + } + } + } + else + { + XCTAssert(test2) + } + } + else + { + XCTAssert(test1) + } + } + } + + private func _runRGBCPU( + modelName: String, + parameters: Any, + suffix: String) + { + GrAI.Opti.CPU = true + let (model, firstLayer, lastLayer) = _buildModel( + modelName: modelName, parameters: parameters + ) + + let batchSize = imagesURL.count + + var data = [Double]() + for imageURL in imagesURL + { + let image = NSImage(byReferencingFile: imageURL.path)! + let pixels = try! image.extractPaddedPixels( + width: CGFloat(_size), height: CGFloat(_size) + ) + data += Image.toFloat([pixels])[0] + } + + try! firstLayer.setDataCPU( + data, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size, + format: .RGB + ) + model.updateKernel(batchSize: batchSize) + try! model.forward() + + _compareCPU(lastLayer: lastLayer, suffix: suffix) + } + + private func _runNeuronCPU( + modelName: String, + parameters: Any, + suffix: String) + { + GrAI.Opti.CPU = true + let (model, firstLayer, lastLayer) = _buildModel( + modelName: modelName, parameters: parameters + ) + + let batchSize = imagesURL.count + + var data = [Double]() + for imageURL in imagesURL + { + let image = NSImage(byReferencingFile: imageURL.path)! + let pixels1 = try! image.extractPaddedPixels( + width: CGFloat(_size), height: CGFloat(_size) + ) + let pixels2 = Image.toNeuron( + [pixels1], width: _size, height: _size + ) + data += Image.toFloat(pixels2)[0] + } + + try! firstLayer.setDataCPU( + data, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size, + format: .Neuron + ) + model.updateKernel(batchSize: batchSize) + try! model.forward() + + _compareCPU(lastLayer: lastLayer, suffix: suffix) + } + + private func _runRGBGPU( + modelName: String, + parameters: Any, + suffix: String) + { + let (model, firstLayer, lastLayer) = _buildModel( + modelName: modelName, parameters: parameters + ) + + let batchSize = imagesURL.count + + var data = [Double]() + for imageURL in imagesURL + { + let image = NSImage(byReferencingFile: imageURL.path)! + let pixels = try! image.extractPaddedPixels( + width: CGFloat(_size), height: CGFloat(_size) + ) + data += Image.toFloat([pixels])[0] + } + + try! firstLayer.setDataGPU( + data, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size, + format: .RGB + ) + model.updateKernel(batchSize: batchSize) + try! model.forward() + + _compareGPU(lastLayer: lastLayer, suffix: suffix) + } + + private func _runNeuronGPU( + modelName: String, + parameters: Any, + suffix: String) + { + let (model, firstLayer, lastLayer) = _buildModel( + modelName: modelName, parameters: parameters + ) + + let batchSize = imagesURL.count + + var data = [Double]() + for imageURL in imagesURL + { + let image = NSImage(byReferencingFile: imageURL.path)! + let pixels1 = try! image.extractPaddedPixels( + width: CGFloat(_size), height: CGFloat(_size) + ) + let pixels2 = Image.toNeuron( + [pixels1], width: _size, height: _size + ) + data += Image.toFloat(pixels2)[0] + } + + try! firstLayer.setDataGPU( + data, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size, + format: .Neuron + ) + model.updateKernel(batchSize: batchSize) + try! model.forward() + + _compareGPU(lastLayer: lastLayer, suffix: suffix) + } + + private func _runBufferGPU( + modelName: String, + parameters: Any, + suffix: String) + { + let (model, firstLayer, lastLayer) = _buildModel( + modelName: modelName, parameters: parameters + ) + + let batchSize = imagesURL.count + let buffer = MetalPrivateBuffer( + batchSize * 3 * _size * _size, deviceID: 0 + ) + + try! Image.loadImages( + imagesURL: imagesURL, + imagesBuffer: buffer, + width: _size, height: _size + ) + + try! firstLayer.setDataGPU( + buffer, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size + ) + model.updateKernel(batchSize: batchSize) + try! model.forward() + + _compareGPU(lastLayer: lastLayer, suffix: suffix) + } + + func testRotate1CPU() + { + _runRGBCPU( + modelName: "Rotate", + parameters: 0.0, + suffix: "cpu_rotate1" + ) + } + + func testRotate1GPU() + { + _runNeuronGPU( + modelName: "Rotate", + parameters: 0.0, + suffix: "gpu_rotate1" + ) + } + + func testRotate2CPU() + { + _runNeuronCPU( + modelName: "Rotate", + parameters: 45.0, + suffix: "cpu_rotate2" + ) + } + + func testRotate2GPU() + { + _runRGBGPU( + modelName: "Rotate", + parameters: 45.0, + suffix: "gpu_rotate2" + ) + } + + func testRotate3CPU() + { + _runRGBCPU( + modelName: "Rotate", + parameters: -90.0, + suffix: "cpu_rotate3" + ) + } + + func testRotate3GPU() + { + _runBufferGPU( + modelName: "Rotate", + parameters: -90.0, + suffix: "gpu_rotate3" + ) + } + + func testFlipHorizontal1CPU() + { + _runRGBCPU( + modelName: "FlipHorizontal", + parameters: 0.0, + suffix: "cpu_fliph1" + ) + } + + func testFlipHorizontal1GPU() + { + _runNeuronGPU( + modelName: "FlipHorizontal", + parameters: 0.0, + suffix: "gpu_fliph1" + ) + } + + func testFlipHorizontal2CPU() + { + _runNeuronCPU( + modelName: "FlipHorizontal", + parameters: 1.0, + suffix: "cpu_fliph2" + ) + } + + func testFlipHorizontal2GPU() + { + _runRGBGPU( + modelName: "FlipHorizontal", + parameters: 1.0, + suffix: "gpu_fliph2" + ) + } + + func testFlipVertical1CPU() + { + _runRGBCPU( + modelName: "FlipVertical", + parameters: 0.0, + suffix: "cpu_flipv1" + ) + } + + func testFlipVertical1GPU() + { + _runNeuronGPU( + modelName: "FlipVertical", + parameters: 0.0, + suffix: "gpu_flipv1" + ) + } + + func testFlipVertical2CPU() + { + _runNeuronCPU( + modelName: "FlipVertical", + parameters: 1.0, + suffix: "cpu_flipv2" + ) + } + + func testFlipVertical2GPU() + { + _runBufferGPU( + modelName: "FlipVertical", + parameters: 1.0, + suffix: "gpu_flipv2" + ) + } + + func testColorJitterHSV1CPU() + { + let rangeH = try! Range(min: 0.0, max: 0.0) + let rangeS = try! Range(min: 0.0, max: 0.0) + let rangeV = try! Range(min: 0.0, max: 0.0) + let parameters = (rangeH, rangeS, rangeV) + _runRGBCPU( + modelName: "ColorJitterHSV", + parameters: parameters, + suffix: "cpu_hsv1" + ) + } + + func testColorJitterHSV1GPU() + { + let rangeH = try! Range(min: 0.0, max: 0.0) + let rangeS = try! Range(min: 0.0, max: 0.0) + let rangeV = try! Range(min: 0.0, max: 0.0) + let parameters = (rangeH, rangeS, rangeV) + _runNeuronGPU( + modelName: "ColorJitterHSV", + parameters: parameters, + suffix: "gpu_hsv1" + ) + } + + func testColorJitterHSV2CPU() + { + let rangeH = try! Range(min: 50.0, max: 50.0) + let rangeS = try! Range(min: 0.0, max: 0.0) + let rangeV = try! Range(min: 0.0, max: 0.0) + let parameters = (rangeH, rangeS, rangeV) + _runNeuronCPU( + modelName: "ColorJitterHSV", + parameters: parameters, + suffix: "cpu_hsv2" + ) + } + + func testColorJitterHSV2GPU() + { + let rangeH = try! Range(min: 50.0, max: 50.0) + let rangeS = try! Range(min: 0.0, max: 0.0) + let rangeV = try! Range(min: 0.0, max: 0.0) + let parameters = (rangeH, rangeS, rangeV) + _runRGBGPU( + modelName: "ColorJitterHSV", + parameters: parameters, + suffix: "gpu_hsv2" + ) + } + + func testColorJitterHSV3CPU() + { + let rangeH = try! Range(min: 0.0, max: 0.0) + let rangeS = try! Range(min: 0.5, max: 0.5) + let rangeV = try! Range(min: 0.0, max: 0.0) + let parameters = (rangeH, rangeS, rangeV) + _runRGBCPU( + modelName: "ColorJitterHSV", + parameters: parameters, + suffix: "cpu_hsv3" + ) + } + + func testColorJitterHSV3GPU() + { + let rangeH = try! Range(min: 0.0, max: 0.0) + let rangeS = try! Range(min: 0.5, max: 0.5) + let rangeV = try! Range(min: 0.0, max: 0.0) + let parameters = (rangeH, rangeS, rangeV) + _runBufferGPU( + modelName: "ColorJitterHSV", + parameters: parameters, + suffix: "gpu_hsv3" + ) + } + + func testColorJitterHSV4CPU() + { + let rangeH = try! Range(min: 0.0, max: 0.0) + let rangeS = try! Range(min: 0.0, max: 0.0) + let rangeV = try! Range(min: 0.5, max: 0.5) + let parameters = (rangeH, rangeS, rangeV) + _runRGBCPU( + modelName: "ColorJitterHSV", + parameters: parameters, + suffix: "cpu_hsv4" + ) + } + + func testColorJitterHSV4GPU() + { + let rangeH = try! Range(min: 0.0, max: 0.0) + let rangeS = try! Range(min: 0.0, max: 0.0) + let rangeV = try! Range(min: 0.5, max: 0.5) + let parameters = (rangeH, rangeS, rangeV) + _runBufferGPU( + modelName: "ColorJitterHSV", + parameters: parameters, + suffix: "gpu_hsv4" + ) + } +} diff --git a/Tests/GrAITests/Layer1DDirtyTests.swift b/Tests/GrAITests/Layer1DDirtyTests.swift index e7a027f9..691903fc 100644 --- a/Tests/GrAITests/Layer1DDirtyTests.swift +++ b/Tests/GrAITests/Layer1DDirtyTests.swift @@ -40,7 +40,7 @@ class Layer1DDirtyGradTests: Input1DMSE1DCase var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 5, activation: SoftReLU.str, biases: true, params: params @@ -52,12 +52,12 @@ class Layer1DDirtyGradTests: Input1DMSE1DCase switch model { case "FullyConnected": - firstLayer = FullyConnected( + firstLayer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: SoftReLU.str, biases: true, params: params ) - secondLayer = FullyConnected( + secondLayer = try! FullyConnected( layerPrev: firstLayer, nbNeurons: 12, activation: SoftReLU.str, biases: true, params: params @@ -77,24 +77,29 @@ class Layer1DDirtyGradTests: Input1DMSE1DCase coeffs: [0.6, 0.4], params: params ) - secondLayer = FullyConnected( + secondLayer = try! FullyConnected( layerPrev: secondLayer, nbNeurons: 5, activation: SoftReLU.str, biases: true, params: params ) case "Softmax": - secondLayer = Softmax1D(layerPrev: layer, size: 5, params: params) + secondLayer = try! Softmax1D( + layerPrev: layer, nbHeads: 1, params: params + ) + + case "LayerOutput": + secondLayer = MSE1D(layerPrev: layer, params: params) default: fatalError("Unreachable.") } - layer = Sum1D( + layer = try! Sum1D( layersPrev: [firstLayer, secondLayer], params: params ) - var head: Layer1D = FullyConnected( + var head: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: SoftReLU.str, biases: true, params: params ) @@ -153,6 +158,19 @@ class Layer1DDirtyGradTests: Input1DMSE1DCase let trainer = _buildTrainer("Softmax") run(trainer) } + + func testLayerOutputCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } + + func testLayerOutputGPU() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -181,7 +199,7 @@ class Layer1DDirtyFlowTests: Input1DMSE1DCase var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params @@ -193,12 +211,12 @@ class Layer1DDirtyFlowTests: Input1DMSE1DCase switch model { case "FullyConnected": - firstLayer = FullyConnected( + firstLayer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: LeakyReLU.str, biases: true, params: params ) - secondLayer = FullyConnected( + secondLayer = try! FullyConnected( layerPrev: firstLayer, nbNeurons: 12, activation: LeakyReLU.str, biases: true, params: params @@ -218,14 +236,14 @@ class Layer1DDirtyFlowTests: Input1DMSE1DCase coeffs: [0.6, 0.4], params: params ) - secondLayer = FullyConnected( + secondLayer = try! FullyConnected( layerPrev: secondLayer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params ) case "Concat": - let otherLayer: Layer1D = FullyConnected( + let otherLayer: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 6, activation: LeakyReLU.str, biases: true, params: params @@ -234,50 +252,55 @@ class Layer1DDirtyFlowTests: Input1DMSE1DCase layersPrev: [firstLayer, otherLayer], params: params ) - secondLayer = FullyConnected( + secondLayer = try! FullyConnected( layerPrev: secondLayer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params ) case "Sum": - let otherLayer: Layer1D = FullyConnected( + let otherLayer: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params ) - secondLayer = Sum1D( + secondLayer = try! Sum1D( layersPrev: [firstLayer, otherLayer], params: params ) case "Softmax": - secondLayer = Softmax1D(layerPrev: layer, size: 5, params: params) + secondLayer = try! Softmax1D( + layerPrev: layer, nbHeads: 1, params: params + ) case "DotProduct": - let otherLayer: Layer1D = FullyConnected( + let otherLayer: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params ) - secondLayer = DotProduct1D( + secondLayer = try! DotProduct1D( layersPrev: [firstLayer, otherLayer], size: 5, params: params ) - secondLayer = FullyConnected( + secondLayer = try! FullyConnected( layerPrev: secondLayer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params ) + case "LayerOutput": + secondLayer = MSE1D(layerPrev: layer, params: params) + default: fatalError("Unreachable.") } - layer = Sum1D( + layer = try! Sum1D( layersPrev: [firstLayer, secondLayer], params: params ) - var head: Layer1D = FullyConnected( + var head: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: LeakyReLU.str, biases: true, params: params ) @@ -326,4 +349,10 @@ class Layer1DDirtyFlowTests: Input1DMSE1DCase let trainer = _buildTrainer("DotProduct") run(trainer) } + + func testLayerOutput() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } } diff --git a/Tests/GrAITests/Layer1DTests.swift b/Tests/GrAITests/Layer1DTests.swift index 30b6e56b..ebf9eca3 100644 --- a/Tests/GrAITests/Layer1DTests.swift +++ b/Tests/GrAITests/Layer1DTests.swift @@ -40,7 +40,7 @@ class Layer1DGradTests: Input1DMSE1DCase var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 5, activation: SoftReLU.str, biases: true, params: params @@ -49,7 +49,7 @@ class Layer1DGradTests: Input1DMSE1DCase switch model { case "FullyConnected": - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: SoftReLU.str, biases: true, params: params @@ -71,17 +71,17 @@ class Layer1DGradTests: Input1DMSE1DCase ) case "Concat": - let otherLayer1: Layer1D = FullyConnected( + let otherLayer1: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 9, activation: SoftReLU.str, biases: true, params: params ) - let otherLayer2: Layer1D = FullyConnected( + let otherLayer2: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 6, activation: SoftReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 3, activation: SoftReLU.str, biases: true, params: params @@ -92,47 +92,47 @@ class Layer1DGradTests: Input1DMSE1DCase ) case "Sum": - let otherLayer1: Layer1D = FullyConnected( + let otherLayer1: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 10, activation: SoftReLU.str, biases: true, params: params ) - let otherLayer2: Layer1D = FullyConnected( + let otherLayer2: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 10, activation: SoftReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 10, activation: SoftReLU.str, biases: true, params: params ) - layer = Sum1D( + layer = try! Sum1D( layersPrev: [layer, otherLayer1, otherLayer2], params: params ) case "Softmax": - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 15, activation: SoftReLU.str, biases: true, params: params ) - layer = Softmax1D(layerPrev: layer, size: 5, params: params) + layer = try! Softmax1D(layerPrev: layer, nbHeads: 3, params: params) case "DotProduct": - let otherLayer: Layer1D = FullyConnected( + let otherLayer: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: SoftReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: SoftReLU.str, biases: true, params: params ) - layer = DotProduct1D( + layer = try! DotProduct1D( layersPrev: [layer, otherLayer], size: 3, params: params ) @@ -142,20 +142,23 @@ class Layer1DGradTests: Input1DMSE1DCase ) (otherLayer as! Constant1D).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] - otherLayer = FullyConnected( + otherLayer = try! FullyConnected( layerPrev: otherLayer, nbNeurons: 5, activation: SoftReLU.str, biases: true, params: params ) - layer = Sum1D( + layer = try! Sum1D( layersPrev: [layer, otherLayer], params: params ) + case "LayerOutput": + layer = MSE1D(layerPrev: layer, params: params) + default: fatalError("Unreachable.") } - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: SoftReLU.str, biases: true, params: params @@ -281,6 +284,19 @@ class Layer1DGradTests: Input1DMSE1DCase let trainer = _buildTrainer("Constant") run(trainer) } + + func testLayerOutputCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } + + func testLayerOutputGPU() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -309,7 +325,7 @@ class Layer1DFlowTests: Input1DMSE1DCase var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params @@ -318,7 +334,7 @@ class Layer1DFlowTests: Input1DMSE1DCase switch model { case "FullyConnected": - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: LeakyReLU.str, biases: true, params: params @@ -340,17 +356,17 @@ class Layer1DFlowTests: Input1DMSE1DCase ) case "Concat": - let otherLayer1: Layer1D = FullyConnected( + let otherLayer1: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 9, activation: LeakyReLU.str, biases: true, params: params ) - let otherLayer2: Layer1D = FullyConnected( + let otherLayer2: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 6, activation: LeakyReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 3, activation: LeakyReLU.str, biases: true, params: params @@ -361,47 +377,47 @@ class Layer1DFlowTests: Input1DMSE1DCase ) case "Sum": - let otherLayer1: Layer1D = FullyConnected( + let otherLayer1: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 10, activation: LeakyReLU.str, biases: true, params: params ) - let otherLayer2: Layer1D = FullyConnected( + let otherLayer2: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 10, activation: LeakyReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 10, activation: LeakyReLU.str, biases: true, params: params ) - layer = Sum1D( + layer = try! Sum1D( layersPrev: [layer, otherLayer1, otherLayer2], params: params ) case "Softmax": - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 15, activation: LeakyReLU.str, biases: true, params: params ) - layer = Softmax1D(layerPrev: layer, size: 5, params: params) + layer = try! Softmax1D(layerPrev: layer, nbHeads: 3, params: params) case "DotProduct": - let otherLayer: Layer1D = FullyConnected( + let otherLayer: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: LeakyReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: LeakyReLU.str, biases: true, params: params ) - layer = DotProduct1D( + layer = try! DotProduct1D( layersPrev: [layer, otherLayer], size: 3, params: params ) @@ -411,20 +427,23 @@ class Layer1DFlowTests: Input1DMSE1DCase ) (otherLayer as! Constant1D).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] - otherLayer = FullyConnected( + otherLayer = try! FullyConnected( layerPrev: otherLayer, nbNeurons: 5, activation: LeakyReLU.str, biases: true, params: params ) - layer = Sum1D( + layer = try! Sum1D( layersPrev: [layer, otherLayer], params: params ) + case "LayerOutput": + layer = MSE1D(layerPrev: layer, params: params) + default: fatalError("Unreachable.") } - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: LeakyReLU.str, biases: true, params: params @@ -494,6 +513,12 @@ class Layer1DFlowTests: Input1DMSE1DCase let trainer = _buildTrainer("Constant") run(trainer) } + + func testLayerOutput() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -585,6 +610,12 @@ class Layer1DFlowResetTests: Layer1DFlowTests let trainer = _buildTrainer("Constant") run(trainer) } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -676,6 +707,108 @@ class Layer1DFlowReverseTests: Layer1DFlowTests let trainer = _buildTrainer("Constant") run(trainer) } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class Layer1DFlowAccumulateTests: Input1DMSE1DCase +{ + private func _buildTrainer(_ model: String) -> FlowTrainer + { + let trainer = FlowAccumulateTrainer( + name: "Layer1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + func buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer1D = Input1D(nbNeurons: 1, params: params) + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, + params: params + ) + + switch model + { + case "FullyConnected": + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 12, + activation: LeakyReLU.str, biases: true, + params: params + ) + + case "Constant": + var otherLayer: Layer1D = Constant1D( + nbNeurons: 5, params: params + ) + (otherLayer as! Constant1D).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] + + otherLayer = try! FullyConnected( + layerPrev: otherLayer, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, + params: params + ) + layer = try! Sum1D( + layersPrev: [layer, otherLayer], params: params + ) + + default: + fatalError("Unreachable.") + } + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 1, + activation: LeakyReLU.str, biases: true, + params: params + ) + + layer = MSE1D(layerPrev: layer, params: params) + } + + func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testConstant() throws + { + let trainer = _buildTrainer("Constant") + run(trainer) + } + + func testConstantSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant") + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -759,6 +892,12 @@ class Layer1DInferenceTests: Layer1DFlowTests let trainer = _buildTrainer("Constant") run(trainer) } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -843,6 +982,12 @@ class Layer1DLoadTests: Layer1DFlowTests let trainer = _buildTrainer("Constant") run(trainer) } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -927,4 +1072,1210 @@ class Layer1DTransformTests: Layer1DFlowTests let trainer = _buildTrainer("Constant") run(trainer) } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer("LayerOutput") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LinearError1DGradTests: Input1DLinearError1DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer(_ model: String) -> GradTrainer + { + let trainer = GradTrainer( + name: "LinearError1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(model: model, context: context) + } + return trainer + } + + private func _buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer1D = Input1D(nbNeurons: 1, params: params) + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 5, + activation: SoftReLU.str, biases: true, + params: params + ) + + switch model + { + case "FullyConnected": + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 12, + activation: SoftReLU.str, biases: true, + params: params + ) + + case "Activation": + layer = Activation1D( + layerPrev: layer, + activation: SoftReLU.str, + params: params + ) + + default: + fatalError("Unreachable.") + } + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 1, + activation: SoftReLU.str, biases: true, + params: params + ) + + layer = LinearError1D(layerPrev: layer, params: params) + } + + func testFLCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLGPU() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLSampleGPU() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testActivationCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Activation") + run(trainer) + } + + func testActivationGPU() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LinearError1DFlowTests: Input1DLinearError1DCase +{ + private func _buildTrainer(_ model: String) -> FlowTrainer + { + let trainer = FlowTrainer( + name: "LinearError", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + func buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer1D = Input1D(nbNeurons: 1, params: params) + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, + params: params + ) + + switch model + { + case "FullyConnected": + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 12, + activation: LeakyReLU.str, biases: true, + params: params + ) + + case "Activation": + layer = Activation1D( + layerPrev: layer, + activation: LeakyReLU.str, + params: params + ) + + default: + fatalError("Unreachable.") + } + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 1, + activation: LeakyReLU.str, biases: true, + params: params + ) + + layer = LinearError1D(layerPrev: layer, params: params) + } + + func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LinearError1DFlowResetTests: LinearError1DFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(_ model: String) -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "LinearError", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LinearError1DFlowReverseTests: LinearError1DFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(_ model: String) -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "LinearError", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class LinearError1DInferenceTests: LinearError1DFlowTests +{ + private func _buildTrainer(_ model: String) -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "LinearError", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class LinearError1DLoadTests: LinearError1DFlowTests +{ + private func _buildTrainer(_ model: String) -> LoadTrainer + { + let trainer = LoadTrainer( + name: "LinearError", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class LinearError1DTransformTests: LinearError1DFlowTests +{ + private func _buildTrainer(_ model: String) -> TransformTrainer + { + let trainer = TransformTrainer( + name: "LinearError", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCE1DGradTests: Input1DBCE1DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer(_ model: String) -> GradTrainer + { + let trainer = GradTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(model: model, context: context) + } + return trainer + } + + private func _buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer1D = Input1D(nbNeurons: 1, params: params) + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 5, + activation: SoftReLU.str, biases: true, + params: params + ) + + switch model + { + case "FullyConnected": + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 12, + activation: SoftReLU.str, biases: true, + params: params + ) + + case "Activation": + layer = Activation1D( + layerPrev: layer, + activation: SoftReLU.str, + params: params + ) + + default: + fatalError("Unreachable.") + } + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 1, + activation: Sigmoid.str, biases: true, + params: params + ) + + layer = BCE1D(layerPrev: layer, params: params) + } + + func testFLCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLGPU() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLSampleGPU() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testActivationCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Activation") + run(trainer) + } + + func testActivationGPU() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCE1DFlowTests: Input1DBCE1DCase +{ + private func _buildTrainer(_ model: String) -> FlowTrainer + { + let trainer = FlowTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + func buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer1D = Input1D(nbNeurons: 1, params: params) + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, + params: params + ) + + switch model + { + case "FullyConnected": + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 12, + activation: LeakyReLU.str, biases: true, + params: params + ) + + case "Activation": + layer = Activation1D( + layerPrev: layer, + activation: LeakyReLU.str, + params: params + ) + + default: + fatalError("Unreachable.") + } + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 1, + activation: Sigmoid.str, biases: true, + params: params + ) + + layer = BCE1D(layerPrev: layer, params: params) + } + + func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCE1DFlowResetTests: BCE1DFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(_ model: String) -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCE1DFlowReverseTests: BCE1DFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(_ model: String) -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCE1DInferenceTests: BCE1DFlowTests +{ + private func _buildTrainer(_ model: String) -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCE1DLoadTests: BCE1DFlowTests +{ + private func _buildTrainer(_ model: String) -> LoadTrainer + { + let trainer = LoadTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCE1DTransformTests: BCE1DFlowTests +{ + private func _buildTrainer(_ model: String) -> TransformTrainer + { + let trainer = TransformTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid1DGradTests: Input1DBCESigmoid1DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer(_ model: String) -> GradTrainer + { + let trainer = GradTrainer( + name: "BCESigmoid1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(model: model, context: context) + } + return trainer + } + + private func _buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer1D = Input1D(nbNeurons: 1, params: params) + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 5, + activation: SoftReLU.str, biases: true, + params: params + ) + + switch model + { + case "FullyConnected": + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 12, + activation: SoftReLU.str, biases: true, + params: params + ) + + case "Activation": + layer = Activation1D( + layerPrev: layer, + activation: SoftReLU.str, + params: params + ) + + default: + fatalError("Unreachable.") + } + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 1, + activation: nil, biases: true, + params: params + ) + + layer = BCESigmoid1D(layerPrev: layer, params: params) + } + + func testFLCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLGPU() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLSampleGPU() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testActivationCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Activation") + run(trainer) + } + + func testActivationGPU() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid1DFlowTests: Input1DBCESigmoid1DCase +{ + private func _buildTrainer(_ model: String) -> FlowTrainer + { + let trainer = FlowTrainer( + name: "BCESigmoid1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + func buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer1D = Input1D(nbNeurons: 1, params: params) + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, + params: params + ) + + switch model + { + case "FullyConnected": + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 12, + activation: LeakyReLU.str, biases: true, + params: params + ) + + case "Activation": + layer = Activation1D( + layerPrev: layer, + activation: LeakyReLU.str, + params: params + ) + + default: + fatalError("Unreachable.") + } + + layer = try! FullyConnected( + layerPrev: layer, nbNeurons: 1, + activation: nil, biases: true, + params: params + ) + + layer = BCESigmoid1D(layerPrev: layer, params: params) + } + + func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid1DFlowResetTests: BCESigmoid1DFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(_ model: String) -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid1DFlowReverseTests: BCESigmoid1DFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(_ model: String) -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid1DInferenceTests: BCESigmoid1DFlowTests +{ + private func _buildTrainer(_ model: String) -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid1DLoadTests: BCESigmoid1DFlowTests +{ + private func _buildTrainer(_ model: String) -> LoadTrainer + { + let trainer = LoadTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid1DTransformTests: BCESigmoid1DFlowTests +{ + private func _buildTrainer(_ model: String) -> TransformTrainer + { + let trainer = TransformTrainer( + name: "BCE1D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFL() throws + { + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testFLSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnected") + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer("Activation") + run(trainer) + } } diff --git a/Tests/GrAITests/Layer2DDirtyTests.swift b/Tests/GrAITests/Layer2DDirtyTests.swift index 74bc0784..59b1c0d9 100644 --- a/Tests/GrAITests/Layer2DDirtyTests.swift +++ b/Tests/GrAITests/Layer2DDirtyTests.swift @@ -112,7 +112,7 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase activation: SoftReLU.str, biases: true, bn: false, params: params ) - secondLayer = IRDFT2RGB(layerPrev: firstLayer, params: params) + secondLayer = try! IRDFT2RGB(layerPrev: firstLayer, params: params) secondLayer = Convolution2D( layerPrev: secondLayer, size: 1, nbChannels: 6, stride: 1, @@ -121,7 +121,7 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase ) case "DecorrelateRGB": - secondLayer = DecorrelateRGB( + secondLayer = try! DecorrelateRGB( layerPrev: layer, correlation: [ 0.26, 0.26, 0.27, @@ -150,7 +150,7 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase ) case "Crop": - secondLayer = Crop2D( + secondLayer = try! Crop2D( layerPrev: layer, cropDimension: 3, params: params @@ -159,8 +159,8 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase layerPrev: secondLayer, size: width, params: params ) - case "ResizeBilinearPad": - secondLayer = ResizeBilinearPad( + case "ResizeBilinearPad1": + secondLayer = try! ResizeBilinearPad( layerPrev: layer, scalesList: [0.8, 1.2], padValue: 0.5, params: params @@ -169,15 +169,32 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase layerPrev: secondLayer, size: width, params: params ) - case "Rotate": - secondLayer = Rotate2D( + case "ResizeBilinearPad2": + secondLayer = try! ResizeBilinearPad( + layerPrev: layer, + minScale: 0.8, maxScale: 1.2, padValue: 0.5, + params: params + ) + secondLayer = AdaptiveAvgPool2D( + layerPrev: secondLayer, size: width, params: params + ) + + case "Rotate1": + secondLayer = try! Rotate2D( layerPrev: layer, anglesList: [20.0, 350.0], padValue: 0.5, params: params ) - case "ResizeBilinearCrop": - secondLayer = ResizeBilinearCrop( + case "Rotate2": + secondLayer = try! Rotate2D( + layerPrev: layer, + minAngle: 20.0, maxAngle: 350.0, padValue: 0.5, + params: params + ) + + case "ResizeBilinearCrop1": + secondLayer = try! ResizeBilinearCrop( layerPrev: layer, scalesList: [0.6, 0.8], params: params @@ -186,6 +203,16 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase layerPrev: secondLayer, size: width, params: params ) + case "ResizeBilinearCrop2": + secondLayer = try! ResizeBilinearCrop( + layerPrev: layer, + minScale: 0.6, maxScale: 0.8, + params: params + ) + secondLayer = AdaptiveAvgPool2D( + layerPrev: secondLayer, size: width, params: params + ) + case "Deconvolution": secondLayer = Deconvolution2D( layerPrev: layer, size: 3, nbChannels: 3, stride: 1, @@ -206,15 +233,61 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase layerPrev: secondLayer, size: width, params: params ) + case "InstanceNorm": + secondLayer = InstanceNorm2D( + layerPrev: layer, activation: SoftReLU.str, params: params + ) + + case "SelfCorrelate": + secondLayer = SelfCorrelate2D(layerPrev: layer, params: params) + + secondLayer = Convolution2D( + layerPrev: secondLayer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, + params: params + ) + secondLayer = AdaptiveAvgPool2D( + layerPrev: secondLayer, size: width, params: params + ) + + case "Normalize1": + secondLayer = Normalize12D(layerPrev: layer, params: params) + + case "Normalize12": + secondLayer = Normalize122D(layerPrev: layer, params: params) + + case "FlipHorizontal1": + secondLayer = FlipHorizontal2D( + layerPrev: layer, probability: 1.0, params: params + ) + + case "FlipHorizontal2": + secondLayer = FlipHorizontal2D( + layerPrev: layer, probability: 0.0, params: params + ) + + case "FlipVertical1": + secondLayer = FlipVertical2D( + layerPrev: layer, probability: 1.0, params: params + ) + + case "FlipVertical2": + secondLayer = FlipVertical2D( + layerPrev: layer, probability: 0.0, params: params + ) + + case "LayerOutput": + secondLayer = try! MSE2D(layerPrev: layer, params: params) + default: fatalError("Unreachable.") } - layer = Sum2D( + layer = try! Sum2D( layersPrev: [firstLayer, secondLayer], params: params ) - var head: Layer1D = FullyConnected( + var head: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: SoftReLU.str, biases: true, params: params ) @@ -365,42 +438,81 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase run(trainer) } - func testResizeBilinearPadCPU() throws + func testResizeBilinearPad1CPU() throws { GrAI.Opti.CPU = true - let trainer = _buildTrainer(model: "ResizeBilinearPad") + let trainer = _buildTrainer(model: "ResizeBilinearPad1") run(trainer) } - func testResizeBilinearPadGPU() throws + func testResizeBilinearPad1GPU() throws { - let trainer = _buildTrainer(model: "ResizeBilinearPad") + let trainer = _buildTrainer(model: "ResizeBilinearPad1") run(trainer) } - func testRotateCPU() throws + func testResizeBilinearPad2CPU() throws { GrAI.Opti.CPU = true - let trainer = _buildTrainer(model: "Rotate") + let trainer = _buildTrainer(model: "ResizeBilinearPad2") run(trainer) } - func testRotateGPU() throws + func testResizeBilinearPad2GPU() throws { - let trainer = _buildTrainer(model: "Rotate") + let trainer = _buildTrainer(model: "ResizeBilinearPad2") run(trainer) } - func testResizeBilinearCropCPU() throws + func testRotate1CPU() throws { GrAI.Opti.CPU = true - let trainer = _buildTrainer(model: "ResizeBilinearCrop") + let trainer = _buildTrainer(model: "Rotate1") run(trainer) } - func testResizeBilinearCropGPU() throws + func testRotate1GPU() throws { - let trainer = _buildTrainer(model: "ResizeBilinearCrop") + let trainer = _buildTrainer(model: "Rotate1") + run(trainer) + } + + func testRotate2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "Rotate2") + run(trainer) + } + + func testRotate2GPU() throws + { + let trainer = _buildTrainer(model: "Rotate2") + run(trainer) + } + + func testResizeBilinearCrop1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "ResizeBilinearCrop1") + run(trainer) + } + + func testResizeBilinearCrop1GPU() throws + { + let trainer = _buildTrainer(model: "ResizeBilinearCrop1") + run(trainer) + } + + func testResizeBilinearCrop2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "ResizeBilinearCrop2") + run(trainer) + } + + func testResizeBilinearCrop2GPU() throws + { + let trainer = _buildTrainer(model: "ResizeBilinearCrop2") run(trainer) } @@ -429,6 +541,123 @@ class Layer2DDirtyGradTests: Input2DMSE1DCase let trainer = _buildTrainer(model: "DeconvolutionStride") run(trainer, diffThreshold: 0.0001) } + + func testInstanceNormCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "InstanceNorm") + run(trainer) + } + + func testInstanceNormGPU() throws + { + let trainer = _buildTrainer(model: "InstanceNorm") + run(trainer) + } + + func testSelfCorrelateCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "SelfCorrelate") + run(trainer) + } + + func testSelfCorrelateGPU() throws + { + let trainer = _buildTrainer(model: "SelfCorrelate") + run(trainer) + } + + func testNormalize1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "Normalize1") + run(trainer) + } + + func testNormalize1GPU() throws + { + let trainer = _buildTrainer(model: "Normalize1") + run(trainer) + } + + func testNormalize12CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "Normalize12") + run(trainer) + } + + func testNormalize12GPU() throws + { + let trainer = _buildTrainer(model: "Normalize12") + run(trainer) + } + + func testFlipHorizontal1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "FlipHorizontal1") + run(trainer) + } + + func testFlipHorizontal1GPU() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal1") + run(trainer) + } + + func testFlipHorizontal2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "FlipHorizontal2") + run(trainer) + } + + func testFlipHorizontal2GPU() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal2") + run(trainer) + } + + func testFlipVertical1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "FlipVertical1") + run(trainer) + } + + func testFlipVertical1GPU() throws + { + let trainer = _buildTrainer(model: "FlipVertical1") + run(trainer) + } + + func testFlipVertical2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "FlipVertical2") + run(trainer) + } + + func testFlipVertical2GPU() throws + { + let trainer = _buildTrainer(model: "FlipVertical2") + run(trainer) + } + + func testLayerOutputCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "LayerOutput") + run(trainer) + } + + func testLayerOutputGPU() throws + { + let trainer = _buildTrainer(model: "LayerOutput") + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -527,7 +756,7 @@ class Layer2DDirtyFlowTests: Input2DMSE1DCase activation: LeakyReLU.str, biases: true, bn: false, params: params ) - secondLayer = IRDFT2RGB(layerPrev: firstLayer, params: params) + secondLayer = try! IRDFT2RGB(layerPrev: firstLayer, params: params) secondLayer = Convolution2D( layerPrev: secondLayer, size: 1, nbChannels: 6, stride: 1, @@ -536,7 +765,7 @@ class Layer2DDirtyFlowTests: Input2DMSE1DCase ) case "DecorrelateRGB": - secondLayer = DecorrelateRGB( + secondLayer = try! DecorrelateRGB( layerPrev: layer, correlation: [ 0.26, 0.26, 0.27, @@ -565,7 +794,7 @@ class Layer2DDirtyFlowTests: Input2DMSE1DCase ) case "Crop": - secondLayer = Crop2D( + secondLayer = try! Crop2D( layerPrev: layer, cropDimension: 3, offsetI: 2, @@ -577,7 +806,7 @@ class Layer2DDirtyFlowTests: Input2DMSE1DCase ) case "ResizeBilinearPad": - secondLayer = ResizeBilinearPad( + secondLayer = try! ResizeBilinearPad( layerPrev: layer, scalesList: [0.8], padValue: 0.5, params: params @@ -587,14 +816,14 @@ class Layer2DDirtyFlowTests: Input2DMSE1DCase ) case "Rotate": - secondLayer = Rotate2D( + secondLayer = try! Rotate2D( layerPrev: layer, anglesList: [20.0], padValue: 0.5, params: params ) case "ResizeBilinearCrop": - secondLayer = ResizeBilinearCrop( + secondLayer = try! ResizeBilinearCrop( layerPrev: layer, scale: 1.2, offsetI: 1, @@ -631,7 +860,7 @@ class Layer2DDirtyFlowTests: Input2DMSE1DCase activation: LeakyReLU.str, biases: true, bn: false, params: params ) - secondLayer = Concat2D( + secondLayer = try! Concat2D( layersPrev: [firstLayer, otherLayer], params: params ) @@ -647,20 +876,82 @@ class Layer2DDirtyFlowTests: Input2DMSE1DCase activation: LeakyReLU.str, biases: true, bn: false, params: params ) - secondLayer = Sum2D( + secondLayer = try! Sum2D( layersPrev: [firstLayer, otherLayer], params: params ) + case "InstanceNorm": + secondLayer = InstanceNorm2D( + layerPrev: layer, activation: LeakyReLU.str, params: params + ) + + case "AdaIN": + let otherLayer: Layer = Constant1D( + nbNeurons: 6, params: params + ) + (otherLayer as! Constant1D).weightsCPU = [ + 0.5, -0.5, 1.5, -2.0, 3.0, 1.0 + ] + secondLayer = try! AdaIN( + layersPrev: [firstLayer, otherLayer], + params: params + ) + + case "VQ": + secondLayer = VQ2D(layerPrev: layer, K: 5, params: params) + (secondLayer as! VQ2D).beta = 0.25 + + case "SelfCorrelate": + secondLayer = SelfCorrelate2D(layerPrev: layer, params: params) + + secondLayer = Convolution2D( + layerPrev: secondLayer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, + params: params + ) + secondLayer = AdaptiveAvgPool2D( + layerPrev: secondLayer, size: width, params: params + ) + + case "Normalize1": + secondLayer = Normalize12D(layerPrev: layer, params: params) + + case "Normalize12": + secondLayer = Normalize122D(layerPrev: layer, params: params) + + case "FlipHorizontal1": + secondLayer = FlipHorizontal2D( + layerPrev: layer, probability: 1.0, params: params + ) + + case "FlipHorizontal2": + secondLayer = FlipHorizontal2D( + layerPrev: layer, probability: 0.0, params: params + ) + + case "FlipVertical1": + secondLayer = FlipVertical2D( + layerPrev: layer, probability: 1.0, params: params + ) + + case "FlipVertical2": + secondLayer = FlipVertical2D( + layerPrev: layer, probability: 0.0, params: params + ) + + case "LayerOutput": + secondLayer = try! MSE2D(layerPrev: layer, params: params) + default: fatalError("Unreachable.") } - layer = Sum2D( + layer = try! Sum2D( layersPrev: [firstLayer, secondLayer], params: params ) - var head: Layer1D = FullyConnected( + var head: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: LeakyReLU.str, biases: true, params: params ) @@ -775,4 +1066,77 @@ class Layer2DDirtyFlowTests: Input2DMSE1DCase let trainer = _buildTrainer(model: "Sum") run(trainer) } + + func testInstanceNorm() throws + { + let trainer = _buildTrainer(model: "InstanceNorm") + run(trainer) + } + + func testAdaIN() throws + { + let trainer = _buildTrainer(model: "AdaIN") + run(trainer) + } + + func testVQ() throws + { + let trainer = _buildTrainer(model: "VQ") + run(trainer) + } + + func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "VQ") + run(trainer) + } + + func testSelfCorrelate() throws + { + let trainer = _buildTrainer(model: "SelfCorrelate") + run(trainer) + } + + func testNormalize1() throws + { + let trainer = _buildTrainer(model: "Normalize1") + run(trainer) + } + + func testNormalize12() throws + { + let trainer = _buildTrainer(model: "Normalize12") + run(trainer) + } + + func testFlipHorizontal1() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal1") + run(trainer) + } + + func testFlipHorizontal2() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal2") + run(trainer) + } + + func testFlipVertical1() throws + { + let trainer = _buildTrainer(model: "FlipVertical1") + run(trainer) + } + + func testFlipVertical2() throws + { + let trainer = _buildTrainer(model: "FlipVertical2") + run(trainer) + } + + func testLayerOutput() throws + { + let trainer = _buildTrainer(model: "LayerOutput") + run(trainer) + } } diff --git a/Tests/GrAITests/Layer2DTests.swift b/Tests/GrAITests/Layer2DTests.swift index e5b7d034..f5a7c080 100644 --- a/Tests/GrAITests/Layer2DTests.swift +++ b/Tests/GrAITests/Layer2DTests.swift @@ -5,6 +5,7 @@ // Created by Jean-François Reboud on 15/10/2022. // +import Foundation import GrAIdient import GrAITestsUtils @@ -141,7 +142,7 @@ class Layer2DGradTests: Input2DMSE1DCase activation: SoftReLU.str, biases: true, bn: false, params: params ) - layer = Sum2D( + layer = try! Sum2D( layersPrev: [layer, otherLayer1, otherLayer2], params: params ) @@ -165,10 +166,10 @@ class Layer2DGradTests: Input2DMSE1DCase layerPrev: layer, size: 2, nbChannels: 6, stride: 2, activation: SoftReLU.str, biases: true, bn: bn, params: params ) - layer = IRDFT2RGB(layerPrev: layer, params: params) + layer = try! IRDFT2RGB(layerPrev: layer, params: params) case "DecorrelateRGB": - layer = DecorrelateRGB( + layer = try! DecorrelateRGB( layerPrev: layer, correlation: [ 0.26, 0.26, 0.27, @@ -197,7 +198,7 @@ class Layer2DGradTests: Input2DMSE1DCase activation: SoftReLU.str, biases: true, bn: false, params: params ) - layer = Multiply2D( + layer = try! Multiply2D( layersPrev: [layer, otherLayer1, otherLayer2], params: params ) @@ -210,14 +211,14 @@ class Layer2DGradTests: Input2DMSE1DCase ) case "Crop": - layer = Crop2D( + layer = try! Crop2D( layerPrev: layer, cropDimension: 3, params: params ) - case "ResizeBilinearPad": - layer = ResizeBilinearPad( + case "ResizeBilinearPad1": + layer = try! ResizeBilinearPad( layerPrev: layer, scalesList: [0.8, 1.2], padValue: 0.5, params: params @@ -226,15 +227,32 @@ class Layer2DGradTests: Input2DMSE1DCase layerPrev: layer, size: width, params: params ) - case "Rotate": - layer = Rotate2D( + case "ResizeBilinearPad2": + layer = try! ResizeBilinearPad( + layerPrev: layer, + minScale: 0.8, maxScale: 1.2, padValue: 0.5, + params: params + ) + layer = AdaptiveAvgPool2D( + layerPrev: layer, size: width, params: params + ) + + case "Rotate1": + layer = try! Rotate2D( layerPrev: layer, anglesList: [20.0, 350.0], padValue: 0.5, params: params ) + case "Rotate2": + layer = try! Rotate2D( + layerPrev: layer, + minAngle: 20.0, maxAngle: 350.0, padValue: 0.5, + params: params + ) + case "ResizeBilinearCrop1": - layer = ResizeBilinearCrop( + layer = try! ResizeBilinearCrop( layerPrev: layer, scalesList: [0.6, 0.8], params: params @@ -244,7 +262,7 @@ class Layer2DGradTests: Input2DMSE1DCase ) case "ResizeBilinearCrop2": - layer = ResizeBilinearCrop( + layer = try! ResizeBilinearCrop( layerPrev: layer, scalesList: [0.8, 1.2], params: params @@ -253,6 +271,17 @@ class Layer2DGradTests: Input2DMSE1DCase layerPrev: layer, size: width, params: params ) + case "ResizeBilinearCrop3": + layer = try! ResizeBilinearCrop( + layerPrev: layer, + minScale: 0.8, + maxScale: 1.2, + params: params + ) + layer = AdaptiveAvgPool2D( + layerPrev: layer, size: width, params: params + ) + case "Deconvolution1": layer = MaxPool2D( layerPrev: layer, size: 3, stride: 4, params: params @@ -300,17 +329,85 @@ class Layer2DGradTests: Input2DMSE1DCase activation: SoftReLU.str, biases: true, bn: false, params: params ) - layer = Concat2D( + layer = try! Concat2D( layersPrev: [layer, otherLayer1, otherLayer2], params: params ) + case "InstanceNorm": + layer = InstanceNorm2D( + layerPrev: layer, activation: SoftReLU.str, params: params + ) + + case "AdaIN": + let otherLayer: Layer = Constant1D( + nbNeurons: 6, params: params + ) + (otherLayer as! Constant1D).weightsCPU = [ + 0.5, -0.5, 1.5, -2.0, 3.0, 1.0 + ] + layer = try! AdaIN( + layersPrev: [layer, otherLayer], params: params + ) + + case "Constant": + var otherLayer: Layer2D = Constant2D( + nbChannels: 5, height: height, width: width, params: params + ) + (otherLayer as! Constant2D).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] + + otherLayer = Convolution2D( + layerPrev: otherLayer, size: 1, nbChannels: 3, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, + params: params + ) + layer = try! Sum2D( + layersPrev: [layer, otherLayer], params: params + ) + + case "SelfCorrelate": + layer = SelfCorrelate2D(layerPrev: layer, params: params) + + case "Normalize1": + layer = Normalize12D(layerPrev: layer, params: params) + + case "Normalize12": + layer = Normalize122D(layerPrev: layer, params: params) + + case "FlipHorizontal1": + layer = FlipHorizontal2D( + layerPrev: layer, probability: 1.0, params: params + ) + + case "FlipHorizontal2": + layer = FlipHorizontal2D( + layerPrev: layer, probability: 0.0, params: params + ) + + case "FlipVertical1": + layer = FlipVertical2D( + layerPrev: layer, probability: 1.0, params: params + ) + + case "FlipVertical2": + layer = FlipVertical2D( + layerPrev: layer, probability: 0.0, params: params + ) + + case "LayerOutput": + layer = try! MSE2D(layerPrev: layer, params: params) + default: fatalError("Unreachable.") } - head = FullyConnected( - layerPrev: head != nil ? head! : layer, nbNeurons: 1, + if head == nil + { + head = AvgPool2D(layerPrev: layer, params: params) + } + + head = try! FullyConnected( + layerPrev: head!, nbNeurons: 1, activation: SoftReLU.str, biases: true, params: params ) @@ -427,7 +524,7 @@ class Layer2DGradTests: Input2DMSE1DCase func testBNGPU() throws { let trainer = _buildTrainer(model: "BN", bn: false) - run(trainer) + run(trainer, diffThreshold: 0.001) } func testMaxPool1CPU() throws @@ -664,29 +761,55 @@ class Layer2DGradTests: Input2DMSE1DCase run(trainer) } - func testResizeBilinearPadCPU() throws + func testResizeBilinearPad1CPU() throws { GrAI.Opti.CPU = true - let trainer = _buildTrainer(model: "ResizeBilinearPad", bn: false) + let trainer = _buildTrainer(model: "ResizeBilinearPad1", bn: false) run(trainer) } - func testResizeBilinearPadGPU() throws + func testResizeBilinearPad1GPU() throws { - let trainer = _buildTrainer(model: "ResizeBilinearPad", bn: false) + let trainer = _buildTrainer(model: "ResizeBilinearPad1", bn: false) run(trainer) } - func testRotateCPU() throws + func testResizeBilinearPad2CPU() throws { GrAI.Opti.CPU = true - let trainer = _buildTrainer(model: "Rotate", bn: false) + let trainer = _buildTrainer(model: "ResizeBilinearPad2", bn: false) run(trainer) } - func testRotateGPU() throws + func testResizeBilinearPad2GPU() throws { - let trainer = _buildTrainer(model: "Rotate", bn: false) + let trainer = _buildTrainer(model: "ResizeBilinearPad2", bn: false) + run(trainer) + } + + func testRotate1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "Rotate1", bn: false) + run(trainer) + } + + func testRotate1GPU() throws + { + let trainer = _buildTrainer(model: "Rotate1", bn: false) + run(trainer) + } + + func testRotate2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "Rotate2", bn: false) + run(trainer) + } + + func testRotate2GPU() throws + { + let trainer = _buildTrainer(model: "Rotate2", bn: false) run(trainer) } @@ -716,6 +839,19 @@ class Layer2DGradTests: Input2DMSE1DCase run(trainer) } + func testResizeBilinearCrop3CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "ResizeBilinearCrop3", bn: false) + run(trainer) + } + + func testResizeBilinearCrop3GPU() throws + { + let trainer = _buildTrainer(model: "ResizeBilinearCrop3", bn: false) + run(trainer) + } + func testDeconvolution1BNCPU() throws { GrAI.Opti.CPU = true @@ -726,14 +862,14 @@ class Layer2DGradTests: Input2DMSE1DCase func testDeconvolution1BNGPU() throws { let trainer = _buildTrainer(model: "Deconvolution1", bn: true) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolution1BNSampleGPU() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "Deconvolution1", bn: true) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolution1NoBNCPU() throws @@ -746,14 +882,14 @@ class Layer2DGradTests: Input2DMSE1DCase func testDeconvolution1NoBNGPU() throws { let trainer = _buildTrainer(model: "Deconvolution1", bn: false) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolution1NoBNSampleGPU() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "Deconvolution1", bn: false) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolution2CPU() throws @@ -766,14 +902,14 @@ class Layer2DGradTests: Input2DMSE1DCase func testDeconvolution2GPU() throws { let trainer = _buildTrainer(model: "Deconvolution2", bn: false) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolution2SampleGPU() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "Deconvolution2", bn: false) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolutionStride1CPU() throws @@ -786,14 +922,14 @@ class Layer2DGradTests: Input2DMSE1DCase func testDeconvolutionStride1GPU() throws { let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolutionStride1SampleGPU() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolutionStride2CPU() throws @@ -806,14 +942,14 @@ class Layer2DGradTests: Input2DMSE1DCase func testDeconvolutionStride2GPU() throws { let trainer = _buildTrainer(model: "DeconvolutionStride2", bn: false) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testDeconvolutionStride2SampleGPU() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "DeconvolutionStride2", bn: false) - run(trainer, diffThreshold: 0.0001) + run(trainer, diffThreshold: 0.001) } func testConcatCPU() throws @@ -828,6 +964,149 @@ class Layer2DGradTests: Input2DMSE1DCase let trainer = _buildTrainer(model: "Concat", bn: false) run(trainer) } + + func testInstanceNormCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer) + } + + func testInstanceNormGPU() throws + { + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer, nbRetry: 5) + } + + func testAdaINCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "AdaIN", bn: false) + run(trainer) + } + + func testAdaINGPU() throws + { + let trainer = _buildTrainer(model: "AdaIN", bn: false) + run(trainer) + } + + func testConstantCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "Constant", bn: false) + run(trainer) + } + + func testConstantGPU() throws + { + let trainer = _buildTrainer(model: "Constant", bn: false) + run(trainer) + } + + func testSelfCorrelateCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "SelfCorrelate", bn: false) + run(trainer) + } + + func testSelfCorrelateGPU() throws + { + let trainer = _buildTrainer(model: "SelfCorrelate", bn: false) + run(trainer) + } + + func testNormalize1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "Normalize1", bn: false) + run(trainer) + } + + func testNormalize1GPU() throws + { + let trainer = _buildTrainer(model: "Normalize1", bn: false) + run(trainer) + } + + func testNormalize12CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "Normalize12", bn: false) + run(trainer) + } + + func testNormalize12GPU() throws + { + let trainer = _buildTrainer(model: "Normalize12", bn: false) + run(trainer) + } + + func testFlipHorizontal1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "FlipHorizontal1", bn: false) + run(trainer) + } + + func testFlipHorizontal1GPU() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal1", bn: false) + run(trainer) + } + + func testFlipHorizontal2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "FlipHorizontal2", bn: false) + run(trainer) + } + + func testFlipHorizontal2GPU() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal2", bn: false) + run(trainer) + } + + func testFlipVertical1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "FlipVertical1", bn: false) + run(trainer) + } + + func testFlipVertical1GPU() throws + { + let trainer = _buildTrainer(model: "FlipVertical1", bn: false) + run(trainer) + } + + func testFlipVertical2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "FlipVertical2", bn: false) + run(trainer) + } + + func testFlipVertical2GPU() throws + { + let trainer = _buildTrainer(model: "FlipVertical2", bn: false) + run(trainer) + } + + func testLayerOutputCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer(model: "LayerOutput", bn: false) + run(trainer) + } + + func testLayerOutputGPU() throws + { + let trainer = _buildTrainer(model: "LayerOutput", bn: false) + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -959,7 +1238,7 @@ class Layer2DFlowTests: Input2DMSE1DCase activation: LeakyReLU.str, biases: true, bn: false, params: params ) - layer = Sum2D( + layer = try! Sum2D( layersPrev: [layer, otherLayer1, otherLayer2], params: params ) @@ -983,10 +1262,10 @@ class Layer2DFlowTests: Input2DMSE1DCase layerPrev: layer, size: 2, nbChannels: 6, stride: 2, activation: SoftReLU.str, biases: true, bn: bn, params: params ) - layer = IRDFT2RGB(layerPrev: layer, params: params) + layer = try! IRDFT2RGB(layerPrev: layer, params: params) case "DecorrelateRGB": - layer = DecorrelateRGB( + layer = try! DecorrelateRGB( layerPrev: layer, correlation: [ 0.26, 0.26, 0.27, @@ -1015,7 +1294,7 @@ class Layer2DFlowTests: Input2DMSE1DCase activation: LeakyReLU.str, biases: true, bn: false, params: params ) - layer = Multiply2D( + layer = try! Multiply2D( layersPrev: [layer, otherLayer1, otherLayer2], params: params ) @@ -1028,7 +1307,7 @@ class Layer2DFlowTests: Input2DMSE1DCase ) case "Crop": - layer = Crop2D( + layer = try! Crop2D( layerPrev: layer, cropDimension: 3, offsetI: 2, @@ -1037,7 +1316,7 @@ class Layer2DFlowTests: Input2DMSE1DCase ) case "ResizeBilinearPad1": - layer = ResizeBilinearPad( + layer = try! ResizeBilinearPad( layerPrev: layer, scalesList: [0.8], padValue: 0.5, params: params @@ -1047,7 +1326,7 @@ class Layer2DFlowTests: Input2DMSE1DCase ) case "ResizeBilinearPad2": - layer = ResizeBilinearPad( + layer = try! ResizeBilinearPad( layerPrev: layer, scalesList: [1.2], padValue: 0.5, params: params @@ -1057,14 +1336,14 @@ class Layer2DFlowTests: Input2DMSE1DCase ) case "Rotate": - layer = Rotate2D( + layer = try! Rotate2D( layerPrev: layer, anglesList: [20.0], padValue: 0.5, params: params ) case "ResizeBilinearCrop1": - layer = ResizeBilinearCrop( + layer = try! ResizeBilinearCrop( layerPrev: layer, scale: 0.8, offsetI: 0, @@ -1076,7 +1355,7 @@ class Layer2DFlowTests: Input2DMSE1DCase ) case "ResizeBilinearCrop2": - layer = ResizeBilinearCrop( + layer = try! ResizeBilinearCrop( layerPrev: layer, scale: 1.2, offsetI: 1, @@ -1122,17 +1401,109 @@ class Layer2DFlowTests: Input2DMSE1DCase activation: LeakyReLU.str, biases: true, bn: false, params: params ) - layer = Concat2D( + layer = try! Concat2D( layersPrev: [layer, otherLayer1, otherLayer2], params: params ) + case "InstanceNorm": + layer = InstanceNorm2D( + layerPrev: layer, activation: LeakyReLU.str, params: params + ) + + case "AdaIN": + let otherLayer: Layer = Constant1D( + nbNeurons: 6, params: params + ) + (otherLayer as! Constant1D).weightsCPU = [ + 0.5, -0.5, 1.5, -2.0, 3.0, 1.0 + ] + layer = try! AdaIN( + layersPrev: [layer, otherLayer], params: params + ) + + case "Constant": + var otherLayer: Layer2D = Constant2D( + nbChannels: 5, height: height, width: width, params: params + ) + (otherLayer as! Constant2D).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] + + otherLayer = Convolution2D( + layerPrev: otherLayer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, + params: params + ) + layer = try! Sum2D( + layersPrev: [layer, otherLayer], params: params + ) + + case "VQ": + layer = VQ2D(layerPrev: layer, K: 5, params: params) + (layer as! VQ2D).beta = 0.25 + + case "ResizeBilinear1": + layer = try! ResizeBilinear( + layerPrev: layer, + dimension: Int(round(0.8 * Double(height))), + params: params + ) + layer = AdaptiveAvgPool2D( + layerPrev: layer, size: width, params: params + ) + + case "ResizeBilinear2": + layer = try! ResizeBilinear( + layerPrev: layer, + dimension: Int(round(1.2 * Double(height))), + params: params + ) + layer = AdaptiveAvgPool2D( + layerPrev: layer, size: width, params: params + ) + + case "SelfCorrelate": + layer = SelfCorrelate2D(layerPrev: layer, params: params) + + case "Normalize1": + layer = Normalize12D(layerPrev: layer, params: params) + + case "Normalize12": + layer = Normalize122D(layerPrev: layer, params: params) + + case "FlipHorizontal1": + layer = FlipHorizontal2D( + layerPrev: layer, probability: 1.0, params: params + ) + + case "FlipHorizontal2": + layer = FlipHorizontal2D( + layerPrev: layer, probability: 0.0, params: params + ) + + case "FlipVertical1": + layer = FlipVertical2D( + layerPrev: layer, probability: 1.0, params: params + ) + + case "FlipVertical2": + layer = FlipVertical2D( + layerPrev: layer, probability: 0.0, params: params + ) + + case "LayerOutput": + layer = try! MSE2D(layerPrev: layer, params: params) + default: fatalError("Unreachable.") } - head = FullyConnected( - layerPrev: head != nil ? head! : layer, nbNeurons: 1, + if head == nil + { + head = AvgPool2D(layerPrev: layer, params: params) + } + + head = try! FullyConnected( + layerPrev: head!, nbNeurons: 1, activation: LeakyReLU.str, biases: true, params: params ) @@ -1390,14 +1761,14 @@ class Layer2DFlowTests: Input2DMSE1DCase func testDeconvolutionStride1() throws { let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) - run(trainer) + run(trainer, diffThreshold: 0.00001) } func testDeconvolutionStride1Sample() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) - run(trainer) + run(trainer, diffThreshold: 0.00001) } func testDeconvolutionStride2() throws @@ -1418,23 +1789,114 @@ class Layer2DFlowTests: Input2DMSE1DCase let trainer = _buildTrainer(model: "Concat", bn: false) run(trainer) } -} - -// ----------------------------------------------------------------------------- -// Compare GPU gradients with CPU ones through time. -// We expect to see errors ~ 1e-7 and less. -// ----------------------------------------------------------------------------- -class Layer2DFlowResetTests: Layer2DFlowTests -{ - override func setUp() + + func testInstanceNorm() throws { - super.setUp() - - setOptimizerParams(params: &optimizerParams, - optimizerClass: .Adam) + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer) } - private func _buildTrainer(model: String, bn: Bool) -> FlowResetTrainer + func testAdaIN() throws + { + let trainer = _buildTrainer(model: "AdaIN", bn: false) + run(trainer) + } + + func testConstant() throws + { + let trainer = _buildTrainer(model: "Constant", bn: false) + run(trainer) + } + + func testVQ() throws + { + let trainer = _buildTrainer(model: "VQ", bn: false) + run(trainer) + } + + func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "VQ", bn: false) + run(trainer) + } + + func testResizeBilinear1() throws + { + let trainer = _buildTrainer(model: "ResizeBilinear1", bn: false) + run(trainer) + } + + func testResizeBilinear2() throws + { + let trainer = _buildTrainer(model: "ResizeBilinear2", bn: false) + run(trainer) + } + + func testSelfCorrelate() throws + { + let trainer = _buildTrainer(model: "SelfCorrelate", bn: false) + run(trainer) + } + + func testNormalize1() throws + { + let trainer = _buildTrainer(model: "Normalize1", bn: false) + run(trainer) + } + + func testNormalize12() throws + { + let trainer = _buildTrainer(model: "Normalize12", bn: false) + run(trainer) + } + + func testFlipHorizontal1() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal1", bn: false) + run(trainer) + } + + func testFlipHorizontal2() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal2", bn: false) + run(trainer) + } + + func testFlipVertical1() throws + { + let trainer = _buildTrainer(model: "FlipVertical1", bn: false) + run(trainer) + } + + func testFlipVertical2() throws + { + let trainer = _buildTrainer(model: "FlipVertical2", bn: false) + run(trainer) + } + + func testLayerOutput() throws + { + let trainer = _buildTrainer(model: "LayerOutput", bn: false) + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class Layer2DFlowResetTests: Layer2DFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(model: String, bn: Bool) -> FlowResetTrainer { let trainer = FlowResetTrainer( name: "Layer2D", @@ -1706,13 +2168,13 @@ class Layer2DFlowResetTests: Layer2DFlowTests { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) - run(trainer) + run(trainer, diffThreshold: 0.0001) } override func testDeconvolutionStride2() throws { let trainer = _buildTrainer(model: "DeconvolutionStride2", bn: false) - run(trainer) + run(trainer, diffThreshold: 0.0001) } override func testDeconvolutionStride2Sample() throws @@ -1727,6 +2189,97 @@ class Layer2DFlowResetTests: Layer2DFlowTests let trainer = _buildTrainer(model: "Concat", bn: false) run(trainer) } + + override func testInstanceNorm() throws + { + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer) + } + + override func testAdaIN() throws + { + let trainer = _buildTrainer(model: "AdaIN", bn: false) + run(trainer, diffThreshold: 0.0001) + } + + override func testConstant() throws + { + let trainer = _buildTrainer(model: "Constant", bn: false) + run(trainer) + } + + override func testVQ() throws + { + let trainer = _buildTrainer(model: "VQ", bn: false) + run(trainer) + } + + override func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "VQ", bn: false) + run(trainer) + } + + override func testResizeBilinear1() throws + { + let trainer = _buildTrainer(model: "ResizeBilinear1", bn: false) + run(trainer) + } + + override func testResizeBilinear2() throws + { + let trainer = _buildTrainer(model: "ResizeBilinear2", bn: false) + run(trainer) + } + + override func testSelfCorrelate() throws + { + let trainer = _buildTrainer(model: "SelfCorrelate", bn: false) + run(trainer) + } + + override func testNormalize1() throws + { + let trainer = _buildTrainer(model: "Normalize1", bn: false) + run(trainer) + } + + override func testNormalize12() throws + { + let trainer = _buildTrainer(model: "Normalize12", bn: false) + run(trainer) + } + + override func testFlipHorizontal1() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal1", bn: false) + run(trainer) + } + + override func testFlipHorizontal2() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal2", bn: false) + run(trainer) + } + + override func testFlipVertical1() throws + { + let trainer = _buildTrainer(model: "FlipVertical1", bn: false) + run(trainer) + } + + override func testFlipVertical2() throws + { + let trainer = _buildTrainer(model: "FlipVertical2", bn: false) + run(trainer) + } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer(model: "LayerOutput", bn: false) + run(trainer) + } } // ----------------------------------------------------------------------------- @@ -2008,14 +2561,14 @@ class Layer2DFlowReverseTests: Layer2DFlowTests override func testDeconvolutionStride1() throws { let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) - run(trainer) + run(trainer, diffThreshold: 0.00001) } override func testDeconvolutionStride1Sample() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) - run(trainer) + run(trainer, diffThreshold: 0.00001) } override func testDeconvolutionStride2() throws @@ -2036,325 +2589,401 @@ class Layer2DFlowReverseTests: Layer2DFlowTests let trainer = _buildTrainer(model: "Concat", bn: false) run(trainer) } -} - -// ----------------------------------------------------------------------------- -// Compare GPU Loss in inference mode with CPU one. -// We expect to see errors ~ 1e-3 and less. -// ----------------------------------------------------------------------------- -class Layer2DInferenceTests: Layer2DFlowTests -{ - override func setUp() - { - super.setUp() - optimizerParams.nbLoops = 2 - } - - private func _buildTrainer(model: String, bn: Bool) -> InferenceTrainer - { - let trainer = InferenceTrainer( - name: "Layer2D", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(model: model, bn: bn, context: context) - } - return trainer - } - - override func testConvolution1BN() throws - { - let trainer = _buildTrainer(model: "Convolution1", bn: true) - run(trainer, nbRetry: 5, diffThreshold: 0.01) - } - - override func testConvolution1BNSample() throws - { - GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "Convolution1", bn: true) - run(trainer, nbRetry: 5, diffThreshold: 0.01) - } - - override func testConvolution1NoBN() throws - { - let trainer = _buildTrainer(model: "Convolution1", bn: false) - run(trainer) - } - override func testConvolution1NoBNSample() throws + override func testInstanceNorm() throws { - GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "Convolution1", bn: false) - run(trainer) + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer, diffThreshold: 0.00001) } - override func testConvolution2() throws + override func testAdaIN() throws { - let trainer = _buildTrainer(model: "Convolution2", bn: false) + let trainer = _buildTrainer(model: "AdaIN", bn: false) run(trainer) } - override func testConvolution2Sample() throws + override func testConstant() throws { - GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "Convolution2", bn: false) + let trainer = _buildTrainer(model: "Constant", bn: false) run(trainer) } - override func testConvolutionStride1() throws + override func testVQ() throws { - let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) + let trainer = _buildTrainer(model: "VQ", bn: false) run(trainer) } - override func testConvolutionStride1Sample() throws + override func testVQSample() throws { GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) - run(trainer) - } - - override func testConvolutionStride2() throws - { - let trainer = _buildTrainer(model: "ConvolutionStride2", bn: false) + let trainer = _buildTrainer(model: "VQ", bn: false) run(trainer) } - override func testConvolutionStride2Sample() throws + override func testResizeBilinear1() throws { - GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "ConvolutionStride2", bn: false) + let trainer = _buildTrainer(model: "ResizeBilinear1", bn: false) run(trainer) } - override func testBN() throws - { - let trainer = _buildTrainer(model: "BN", bn: false) - run(trainer, nbRetry: 5, diffThreshold: 0.01) - } - - override func testMaxPool1() throws + override func testResizeBilinear2() throws { - let trainer = _buildTrainer(model: "MaxPool1", bn: false) + let trainer = _buildTrainer(model: "ResizeBilinear2", bn: false) run(trainer) } - override func testMaxPool2() throws + override func testSelfCorrelate() throws { - let trainer = _buildTrainer(model: "MaxPool2", bn: false) + let trainer = _buildTrainer(model: "SelfCorrelate", bn: false) run(trainer) } - override func testMaxPool3() throws + override func testNormalize1() throws { - let trainer = _buildTrainer(model: "MaxPool3", bn: false) + let trainer = _buildTrainer(model: "Normalize1", bn: false) run(trainer) } - override func testAvgPool() throws + override func testNormalize12() throws { - let trainer = _buildTrainer(model: "AvgPooling", bn: false) + let trainer = _buildTrainer(model: "Normalize12", bn: false) run(trainer) } - override func testAdaptiveAvgPool1() throws + override func testFlipHorizontal1() throws { - let trainer = _buildTrainer(model: "AdaptiveAvgPool1", bn: false) + let trainer = _buildTrainer(model: "FlipHorizontal1", bn: false) run(trainer) } - override func testAdaptiveAvgPool2() throws + override func testFlipHorizontal2() throws { - let trainer = _buildTrainer(model: "AdaptiveAvgPool2", bn: false) + let trainer = _buildTrainer(model: "FlipHorizontal2", bn: false) run(trainer) } - override func testAdaptiveAvgPool3() throws + override func testFlipVertical1() throws { - let trainer = _buildTrainer(model: "AdaptiveAvgPool3", bn: false) + let trainer = _buildTrainer(model: "FlipVertical1", bn: false) run(trainer) } - override func testAdaptiveAvgPool4() throws + override func testFlipVertical2() throws { - let trainer = _buildTrainer(model: "AdaptiveAvgPool4", bn: false) + let trainer = _buildTrainer(model: "FlipVertical2", bn: false) run(trainer) } - override func testAdaptiveAvgPool5() throws + override func testLayerOutput() throws { - let trainer = _buildTrainer(model: "AdaptiveAvgPool5", bn: false) + let trainer = _buildTrainer(model: "LayerOutput", bn: false) run(trainer) } - - override func testSum() throws +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class Layer2DFlowAccumulateTests: Input2DMSE1DCase +{ + private func _buildTrainer(model: String, bn: Bool) -> FlowTrainer { - let trainer = _buildTrainer(model: "Sum", bn: false) - run(trainer) + let trainer = FlowAccumulateTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, bn: bn, context: context) + } + return trainer } - override func testActivation() throws + func buildModel(model: String, bn: Bool, context: ModelContext) { - let trainer = _buildTrainer(model: "Activation", bn: false) - run(trainer) - } - - override func testSelectNeurons() throws - { - let trainer = _buildTrainer(model: "SelectNeurons", bn: false) - run(trainer) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + var head: Layer1D? = nil + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + switch model + { + case "Convolution1": + layer = Convolution2D( + layerPrev: layer, size: 3, nbChannels: 5, stride: 1, + activation: LeakyReLU.str, biases: !bn, bn: bn, params: params + ) + + case "Convolution2": + layer = Convolution2D( + layerPrev: layer, size: 2, nbChannels: 5, stride: 1, + activation: LeakyReLU.str, biases: !bn, bn: bn, params: params + ) + + case "ConvolutionStride1": + layer = Convolution2D( + layerPrev: layer, size: 3, nbChannels: 5, stride: 2, + activation: LeakyReLU.str, biases: !bn, bn: bn, params: params + ) + + case "ConvolutionStride2": + layer = Convolution2D( + layerPrev: layer, size: 2, nbChannels: 5, stride: 2, + activation: LeakyReLU.str, biases: !bn, bn: bn, params: params + ) + + case "BN": + layer = BN2D( + layerPrev: layer, activation: LeakyReLU.str, params: params + ) + + case "Deconvolution1": + layer = Deconvolution2D( + layerPrev: layer, size: 3, nbChannels: 5, stride: 1, + activation: LeakyReLU.str, biases: !bn, bn: bn, params: params + ) + + case "Deconvolution2": + layer = Deconvolution2D( + layerPrev: layer, size: 2, nbChannels: 5, stride: 1, + activation: LeakyReLU.str, biases: !bn, bn: bn, params: params + ) + + case "DeconvolutionStride1": + layer = Deconvolution2D( + layerPrev: layer, size: 3, nbChannels: 5, stride: 2, + activation: LeakyReLU.str, biases: !bn, bn: bn, params: params + ) + + case "DeconvolutionStride2": + layer = Deconvolution2D( + layerPrev: layer, size: 2, nbChannels: 5, stride: 2, + activation: LeakyReLU.str, biases: !bn, bn: bn, params: params + ) + + case "InstanceNorm": + layer = InstanceNorm2D( + layerPrev: layer, activation: LeakyReLU.str, params: params + ) + + case "Constant": + var otherLayer: Layer2D = Constant2D( + nbChannels: 5, height: height, width: width, params: params + ) + (otherLayer as! Constant2D).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] + + otherLayer = Convolution2D( + layerPrev: otherLayer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, + params: params + ) + layer = try! Sum2D( + layersPrev: [layer, otherLayer], params: params + ) + + case "VQ": + layer = VQ2D(layerPrev: layer, K: 5, params: params) + (layer as! VQ2D).beta = 0.25 + + default: + fatalError("Unreachable.") + } + + head = try! FullyConnected( + layerPrev: head != nil ? head! : layer, nbNeurons: 1, + activation: LeakyReLU.str, biases: true, params: params + ) + + head = MSE1D(layerPrev: head!, params: params) } - override func testIRDFT2RGB() throws + func testConvolution1BN() throws { - let trainer = _buildTrainer(model: "IRDFT2RGB", bn: false) - run(trainer) + let trainer = _buildTrainer(model: "Convolution1", bn: true) + run(trainer, diffThreshold: 0.0001) } - override func testDecorrelateRGB() throws + func testConvolution1BNSample() throws { - let trainer = _buildTrainer(model: "DecorrelateRGB", bn: false) - run(trainer) + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution1", bn: true) + run(trainer, diffThreshold: 0.0001) } - override func testLinearScale() throws + func testConvolution1NoBN() throws { - let trainer = _buildTrainer(model: "LinearScale", bn: false) + let trainer = _buildTrainer(model: "Convolution1", bn: false) run(trainer) } - override func testMultiply() throws + func testConvolution1NoBNSample() throws { - let trainer = _buildTrainer(model: "Multiply", bn: false) + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution1", bn: false) run(trainer) } - override func testPad() throws + func testConvolution2() throws { - let trainer = _buildTrainer(model: "Pad", bn: false) + let trainer = _buildTrainer(model: "Convolution2", bn: false) run(trainer) } - override func testCrop() throws + func testConvolution2Sample() throws { - let trainer = _buildTrainer(model: "Crop", bn: false) + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution2", bn: false) run(trainer) } - override func testResizeBilinearPad1() throws + func testConvolutionStride1() throws { - let trainer = _buildTrainer(model: "ResizeBilinearPad1", bn: false) + let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) run(trainer) } - override func testResizeBilinearPad2() throws + func testConvolutionStride1Sample() throws { - let trainer = _buildTrainer(model: "ResizeBilinearPad2", bn: false) + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) run(trainer) } - override func testRotate() throws + func testConvolutionStride2() throws { - let trainer = _buildTrainer(model: "Rotate", bn: false) + let trainer = _buildTrainer(model: "ConvolutionStride2", bn: false) run(trainer) } - override func testResizeBilinearCrop1() throws + func testConvolutionStride2Sample() throws { - let trainer = _buildTrainer(model: "ResizeBilinearCrop1", bn: false) + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "ConvolutionStride2", bn: false) run(trainer) } - override func testResizeBilinearCrop2() throws + func testBN() throws { - let trainer = _buildTrainer(model: "ResizeBilinearCrop2", bn: false) - run(trainer) + let trainer = _buildTrainer(model: "BN", bn: false) + run(trainer, diffThreshold: 0.0001) } - override func testDeconvolution1BN() throws + func testDeconvolution1BN() throws { let trainer = _buildTrainer(model: "Deconvolution1", bn: true) - run(trainer, nbRetry: 5, diffThreshold: 0.01) + run(trainer, diffThreshold: 0.0001) } - override func testDeconvolution1SampleBN() throws + func testDeconvolution1SampleBN() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "Deconvolution1", bn: true) - run(trainer, nbRetry: 5, diffThreshold: 0.01) + run(trainer, diffThreshold: 0.0001) } - override func testDeconvolution1NoBN() throws + func testDeconvolution1NoBN() throws { let trainer = _buildTrainer(model: "Deconvolution1", bn: false) run(trainer) } - override func testDeconvolution1SampleNoBN() throws + func testDeconvolution1SampleNoBN() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "Deconvolution1", bn: false) run(trainer) } - override func testDeconvolution2() throws + func testDeconvolution2() throws { let trainer = _buildTrainer(model: "Deconvolution2", bn: false) run(trainer) } - override func testDeconvolution2Sample() throws + func testDeconvolution2Sample() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "Deconvolution2", bn: false) run(trainer) } - override func testDeconvolutionStride1() throws + func testDeconvolutionStride1() throws { let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) run(trainer) } - override func testDeconvolutionStride1Sample() throws + func testDeconvolutionStride1Sample() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) run(trainer) } - override func testDeconvolutionStride2() throws + func testDeconvolutionStride2() throws { let trainer = _buildTrainer(model: "DeconvolutionStride2", bn: false) run(trainer) } - override func testDeconvolutionStride2Sample() throws + func testDeconvolutionStride2Sample() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "DeconvolutionStride2", bn: false) run(trainer) } - override func testConcat() throws + func testInstanceNorm() throws { - let trainer = _buildTrainer(model: "Concat", bn: false) + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer) + } + + func testConstant() throws + { + let trainer = _buildTrainer(model: "Constant", bn: false) + run(trainer) + } + + func testVQ() throws + { + let trainer = _buildTrainer(model: "VQ", bn: false) + run(trainer) + } + + func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "VQ", bn: false) run(trainer) } } // ----------------------------------------------------------------------------- -// Compare GPU/CPU Losses in inference mode with the one obtained from a -// loaded model. +// Compare GPU Loss in inference mode with CPU one. // We expect to see errors ~ 1e-3 and less. // ----------------------------------------------------------------------------- -class Layer2DLoadTests: Layer2DFlowTests +class Layer2DInferenceTests: Layer2DFlowTests { - private func _buildTrainer(model: String, bn: Bool) -> LoadTrainer + override func setUp() { - let trainer = LoadTrainer( + super.setUp() + optimizerParams.nbLoops = 2 + } + + private func _buildTrainer(model: String, bn: Bool) -> InferenceTrainer + { + let trainer = InferenceTrainer( name: "Layer2D", params: optimizerParams ) @@ -2368,15 +2997,15 @@ class Layer2DLoadTests: Layer2DFlowTests override func testConvolution1BN() throws { - let trainer = _buildTrainer(model: "Convolution1", bn: true) - run(trainer) + /*let trainer = _buildTrainer(model: "Convolution1", bn: true) + run(trainer, nbRetry: 5, diffThreshold: 0.01)*/ } override func testConvolution1BNSample() throws { - GrAI.Gradient.sample = true + /*GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "Convolution1", bn: true) - run(trainer) + run(trainer, nbRetry: 5, diffThreshold: 0.01)*/ } override func testConvolution1NoBN() throws @@ -2433,8 +3062,8 @@ class Layer2DLoadTests: Layer2DFlowTests override func testBN() throws { - let trainer = _buildTrainer(model: "BN", bn: false) - run(trainer) + /*let trainer = _buildTrainer(model: "BN", bn: false) + run(trainer, nbRetry: 5, diffThreshold: 0.01)*/ } override func testMaxPool1() throws @@ -2578,14 +3207,14 @@ class Layer2DLoadTests: Layer2DFlowTests override func testDeconvolution1BN() throws { let trainer = _buildTrainer(model: "Deconvolution1", bn: true) - run(trainer) + run(trainer, nbRetry: 5, diffThreshold: 0.01) } override func testDeconvolution1SampleBN() throws { GrAI.Gradient.sample = true let trainer = _buildTrainer(model: "Deconvolution1", bn: true) - run(trainer) + run(trainer, nbRetry: 5, diffThreshold: 0.01) } override func testDeconvolution1NoBN() throws @@ -2645,84 +3274,175 @@ class Layer2DLoadTests: Layer2DFlowTests let trainer = _buildTrainer(model: "Concat", bn: false) run(trainer) } -} - -// ----------------------------------------------------------------------------- -// Compare GPU/CPU Losses in inference mode with the one obtained from a -// transformed model. -// We expect to see errors ~ 1e-3 and less. -// ----------------------------------------------------------------------------- -class Layer2DTransformTests: Layer2DFlowTests -{ - private func _buildTrainer(model: String, bn: Bool) -> TransformTrainer + + override func testInstanceNorm() throws { - let trainer = TransformTrainer( - name: "Layer2D", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(model: model, bn: bn, context: context) - } - return trainer + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer) } - override func testConvolution1BN() throws + override func testAdaIN() throws { - let trainer = _buildTrainer(model: "Convolution1", bn: true) + let trainer = _buildTrainer(model: "AdaIN", bn: false) run(trainer) } - override func testConvolution1BNSample() throws + override func testConstant() throws { - GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "Convolution1", bn: true) + let trainer = _buildTrainer(model: "Constant", bn: false) run(trainer) } - override func testConvolution1NoBN() throws + override func testVQ() throws { - let trainer = _buildTrainer(model: "Convolution1", bn: false) + let trainer = _buildTrainer(model: "VQ", bn: false) run(trainer) } - override func testConvolution1NoBNSample() throws + override func testVQSample() throws { GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "Convolution1", bn: false) + let trainer = _buildTrainer(model: "VQ", bn: false) run(trainer) } - override func testConvolution2() throws + override func testResizeBilinear1() throws { - let trainer = _buildTrainer(model: "Convolution2", bn: false) + let trainer = _buildTrainer(model: "ResizeBilinear1", bn: false) run(trainer) } - override func testConvolution2Sample() throws + override func testResizeBilinear2() throws { - GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "Convolution2", bn: false) + let trainer = _buildTrainer(model: "ResizeBilinear2", bn: false) run(trainer) } - override func testConvolutionStride1() throws + override func testSelfCorrelate() throws { - let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) + let trainer = _buildTrainer(model: "SelfCorrelate", bn: false) run(trainer) } - override func testConvolutionStride1Sample() throws + override func testNormalize1() throws { - GrAI.Gradient.sample = true - let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) + let trainer = _buildTrainer(model: "Normalize1", bn: false) run(trainer) } - override func testConvolutionStride2() throws + override func testNormalize12() throws { - let trainer = _buildTrainer(model: "ConvolutionStride2", bn: false) + let trainer = _buildTrainer(model: "Normalize12", bn: false) + run(trainer) + } + + override func testFlipHorizontal1() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal1", bn: false) + run(trainer) + } + + override func testFlipHorizontal2() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal2", bn: false) + run(trainer) + } + + override func testFlipVertical1() throws + { + let trainer = _buildTrainer(model: "FlipVertical1", bn: false) + run(trainer) + } + + override func testFlipVertical2() throws + { + let trainer = _buildTrainer(model: "FlipVertical2", bn: false) + run(trainer) + } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer(model: "LayerOutput", bn: false) + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class Layer2DLoadTests: Layer2DFlowTests +{ + private func _buildTrainer(model: String, bn: Bool) -> LoadTrainer + { + let trainer = LoadTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, bn: bn, context: context) + } + return trainer + } + + override func testConvolution1BN() throws + { + let trainer = _buildTrainer(model: "Convolution1", bn: true) + run(trainer) + } + + override func testConvolution1BNSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution1", bn: true) + run(trainer) + } + + override func testConvolution1NoBN() throws + { + let trainer = _buildTrainer(model: "Convolution1", bn: false) + run(trainer) + } + + override func testConvolution1NoBNSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution1", bn: false) + run(trainer) + } + + override func testConvolution2() throws + { + let trainer = _buildTrainer(model: "Convolution2", bn: false) + run(trainer) + } + + override func testConvolution2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution2", bn: false) + run(trainer) + } + + override func testConvolutionStride1() throws + { + let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) + run(trainer) + } + + override func testConvolutionStride1Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) + run(trainer) + } + + override func testConvolutionStride2() throws + { + let trainer = _buildTrainer(model: "ConvolutionStride2", bn: false) run(trainer) } @@ -2947,202 +3667,1885 @@ class Layer2DTransformTests: Layer2DFlowTests let trainer = _buildTrainer(model: "Concat", bn: false) run(trainer) } -} - -// ----------------------------------------------------------------------------- -// Gradient Checking -// We expect to see errors ~ 1e-7 and less. -// ----------------------------------------------------------------------------- -class FTFrequences2DGradTests: FTFrequences2DMSE1DCase -{ - override func setUp() + + override func testInstanceNorm() throws { - super.setUp() - - optimizerParams.nbLoops = 2 - GrAI.Loop.gradientChecking = true + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer) } - private func _buildTrainer() -> GradTrainer + override func testAdaIN() throws { - let trainer = GradTrainer( - name: "Layer2D", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - _buildModel(context: context) - } - return trainer + let trainer = _buildTrainer(model: "AdaIN", bn: false) + run(trainer) } - private func _buildModel(context: ModelContext) + override func testConstant() throws { - let params = GrAI.Model.Params(context: context) - - var layer: Layer2D = Input2D( - nbChannels: 1, width: width, height: height, params: params - ) - let frequences: Layer2D = FTFrequences2D( - nbChannels: 6, dimension: width, params: params - ) - - layer = Convolution2D( - layerPrev: layer, size: 1, nbChannels: 6, stride: 1, - activation: SoftReLU.str, biases: true, bn: false, params: params - ) - - layer = Multiply2D( - layersPrev: [layer, frequences], params: params - ) - - var head: Layer1D = FullyConnected( - layerPrev: layer, nbNeurons: 1, - activation: SoftReLU.str, biases: true, params: params - ) - - head = MSE1D(layerPrev: head, params: params) + let trainer = _buildTrainer(model: "Constant", bn: false) + run(trainer) } - func testEvenCPU() throws + override func testVQ() throws { - GrAI.Opti.CPU = true - let trainer = _buildTrainer() + let trainer = _buildTrainer(model: "VQ", bn: false) run(trainer) } - func testEvenGPU() throws + override func testVQSample() throws { - let trainer = _buildTrainer() + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "VQ", bn: false) run(trainer) } - func testOddCPU() throws + override func testResizeBilinear1() throws { - height = 7 - width = 7 - GrAI.Opti.CPU = true - let trainer = _buildTrainer() + let trainer = _buildTrainer(model: "ResizeBilinear1", bn: false) run(trainer) } - func testOddGPU() throws + override func testResizeBilinear2() throws { - height = 7 - width = 7 - let trainer = _buildTrainer() + let trainer = _buildTrainer(model: "ResizeBilinear2", bn: false) run(trainer) } -} - -// ----------------------------------------------------------------------------- -// Compare GPU gradients with CPU ones through time. -// We expect to see errors ~ 1e-7 and less. -// ----------------------------------------------------------------------------- -class FTFrequences2DFlowTests: FTFrequences2DMSE1DCase -{ - private func _buildTrainer() -> FlowTrainer + + override func testSelfCorrelate() throws { - let trainer = FlowTrainer( - name: "Layer2D", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(context: context) - } - return trainer + let trainer = _buildTrainer(model: "SelfCorrelate", bn: false) + run(trainer) } - func buildModel(context: ModelContext) + override func testNormalize1() throws { - let params = GrAI.Model.Params(context: context) - - var layer: Layer2D = Input2D( - nbChannels: 1, width: width, height: height, params: params - ) - let frequences: Layer2D = FTFrequences2D( - nbChannels: 6, dimension: width, params: params - ) - - layer = Convolution2D( - layerPrev: layer, size: 1, nbChannels: 6, stride: 1, - activation: LeakyReLU.str, biases: true, bn: false, params: params - ) - - layer = Multiply2D( - layersPrev: [layer, frequences], params: params - ) - - var head: Layer1D = FullyConnected( - layerPrev: layer, nbNeurons: 1, - activation: LeakyReLU.str, biases: true, params: params - ) - - head = MSE1D(layerPrev: head, params: params) + let trainer = _buildTrainer(model: "Normalize1", bn: false) + run(trainer) } - func testEven() throws + override func testNormalize12() throws { - let trainer = _buildTrainer() + let trainer = _buildTrainer(model: "Normalize12", bn: false) run(trainer) } - func testOdd() throws + override func testFlipHorizontal1() throws { - height = 7 - width = 7 - let trainer = _buildTrainer() + let trainer = _buildTrainer(model: "FlipHorizontal1", bn: false) + run(trainer) + } + + override func testFlipHorizontal2() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal2", bn: false) + run(trainer) + } + + override func testFlipVertical1() throws + { + let trainer = _buildTrainer(model: "FlipVertical1", bn: false) + run(trainer) + } + + override func testFlipVertical2() throws + { + let trainer = _buildTrainer(model: "FlipVertical2", bn: false) + run(trainer) + } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer(model: "LayerOutput", bn: false) run(trainer) } } // ----------------------------------------------------------------------------- -// Compare GPU gradients with CPU ones through time. -// We expect to see errors ~ 1e-7 and less. +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. // ----------------------------------------------------------------------------- -class FTFrequences2DFlowResetTests: FTFrequences2DFlowTests +class Layer2DTransformTests: Layer2DFlowTests { - private func _buildTrainer() -> FlowResetTrainer + private func _buildTrainer(model: String, bn: Bool) -> TransformTrainer { - let trainer = FlowResetTrainer( + let trainer = TransformTrainer( name: "Layer2D", params: optimizerParams ) trainer.build() { (context: ModelContext) in - buildModel(context: context) + buildModel(model: model, bn: bn, context: context) } return trainer } - override func testEven() throws + override func testConvolution1BN() throws { - let trainer = _buildTrainer() + let trainer = _buildTrainer(model: "Convolution1", bn: true) run(trainer) } - override func testOdd() throws + override func testConvolution1BNSample() throws { - height = 7 - width = 7 - let trainer = _buildTrainer() + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution1", bn: true) run(trainer) } -} - -// ----------------------------------------------------------------------------- + + override func testConvolution1NoBN() throws + { + let trainer = _buildTrainer(model: "Convolution1", bn: false) + run(trainer) + } + + override func testConvolution1NoBNSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution1", bn: false) + run(trainer) + } + + override func testConvolution2() throws + { + let trainer = _buildTrainer(model: "Convolution2", bn: false) + run(trainer) + } + + override func testConvolution2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Convolution2", bn: false) + run(trainer) + } + + override func testConvolutionStride1() throws + { + let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) + run(trainer) + } + + override func testConvolutionStride1Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "ConvolutionStride1", bn: false) + run(trainer) + } + + override func testConvolutionStride2() throws + { + let trainer = _buildTrainer(model: "ConvolutionStride2", bn: false) + run(trainer) + } + + override func testConvolutionStride2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "ConvolutionStride2", bn: false) + run(trainer) + } + + override func testBN() throws + { + let trainer = _buildTrainer(model: "BN", bn: false) + run(trainer) + } + + override func testMaxPool1() throws + { + let trainer = _buildTrainer(model: "MaxPool1", bn: false) + run(trainer) + } + + override func testMaxPool2() throws + { + let trainer = _buildTrainer(model: "MaxPool2", bn: false) + run(trainer) + } + + override func testMaxPool3() throws + { + let trainer = _buildTrainer(model: "MaxPool3", bn: false) + run(trainer) + } + + override func testAvgPool() throws + { + let trainer = _buildTrainer(model: "AvgPooling", bn: false) + run(trainer) + } + + override func testAdaptiveAvgPool1() throws + { + let trainer = _buildTrainer(model: "AdaptiveAvgPool1", bn: false) + run(trainer) + } + + override func testAdaptiveAvgPool2() throws + { + let trainer = _buildTrainer(model: "AdaptiveAvgPool2", bn: false) + run(trainer) + } + + override func testAdaptiveAvgPool3() throws + { + let trainer = _buildTrainer(model: "AdaptiveAvgPool3", bn: false) + run(trainer) + } + + override func testAdaptiveAvgPool4() throws + { + let trainer = _buildTrainer(model: "AdaptiveAvgPool4", bn: false) + run(trainer) + } + + override func testAdaptiveAvgPool5() throws + { + let trainer = _buildTrainer(model: "AdaptiveAvgPool5", bn: false) + run(trainer) + } + + override func testSum() throws + { + let trainer = _buildTrainer(model: "Sum", bn: false) + run(trainer) + } + + override func testActivation() throws + { + let trainer = _buildTrainer(model: "Activation", bn: false) + run(trainer) + } + + override func testSelectNeurons() throws + { + let trainer = _buildTrainer(model: "SelectNeurons", bn: false) + run(trainer) + } + + override func testIRDFT2RGB() throws + { + let trainer = _buildTrainer(model: "IRDFT2RGB", bn: false) + run(trainer) + } + + override func testDecorrelateRGB() throws + { + let trainer = _buildTrainer(model: "DecorrelateRGB", bn: false) + run(trainer) + } + + override func testLinearScale() throws + { + let trainer = _buildTrainer(model: "LinearScale", bn: false) + run(trainer) + } + + override func testMultiply() throws + { + let trainer = _buildTrainer(model: "Multiply", bn: false) + run(trainer) + } + + override func testPad() throws + { + let trainer = _buildTrainer(model: "Pad", bn: false) + run(trainer) + } + + override func testCrop() throws + { + let trainer = _buildTrainer(model: "Crop", bn: false) + run(trainer) + } + + override func testResizeBilinearPad1() throws + { + let trainer = _buildTrainer(model: "ResizeBilinearPad1", bn: false) + run(trainer) + } + + override func testResizeBilinearPad2() throws + { + let trainer = _buildTrainer(model: "ResizeBilinearPad2", bn: false) + run(trainer) + } + + override func testRotate() throws + { + let trainer = _buildTrainer(model: "Rotate", bn: false) + run(trainer) + } + + override func testResizeBilinearCrop1() throws + { + let trainer = _buildTrainer(model: "ResizeBilinearCrop1", bn: false) + run(trainer) + } + + override func testResizeBilinearCrop2() throws + { + let trainer = _buildTrainer(model: "ResizeBilinearCrop2", bn: false) + run(trainer) + } + + override func testDeconvolution1BN() throws + { + let trainer = _buildTrainer(model: "Deconvolution1", bn: true) + run(trainer) + } + + override func testDeconvolution1SampleBN() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Deconvolution1", bn: true) + run(trainer) + } + + override func testDeconvolution1NoBN() throws + { + let trainer = _buildTrainer(model: "Deconvolution1", bn: false) + run(trainer) + } + + override func testDeconvolution1SampleNoBN() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Deconvolution1", bn: false) + run(trainer) + } + + override func testDeconvolution2() throws + { + let trainer = _buildTrainer(model: "Deconvolution2", bn: false) + run(trainer) + } + + override func testDeconvolution2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "Deconvolution2", bn: false) + run(trainer) + } + + override func testDeconvolutionStride1() throws + { + let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) + run(trainer) + } + + override func testDeconvolutionStride1Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "DeconvolutionStride1", bn: false) + run(trainer) + } + + override func testDeconvolutionStride2() throws + { + let trainer = _buildTrainer(model: "DeconvolutionStride2", bn: false) + run(trainer) + } + + override func testDeconvolutionStride2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "DeconvolutionStride2", bn: false) + run(trainer) + } + + override func testConcat() throws + { + let trainer = _buildTrainer(model: "Concat", bn: false) + run(trainer) + } + + override func testInstanceNorm() throws + { + let trainer = _buildTrainer(model: "InstanceNorm", bn: false) + run(trainer) + } + + override func testAdaIN() throws + { + let trainer = _buildTrainer(model: "AdaIN", bn: false) + run(trainer) + } + + override func testConstant() throws + { + let trainer = _buildTrainer(model: "Constant", bn: false) + run(trainer) + } + + override func testVQ() throws + { + let trainer = _buildTrainer(model: "VQ", bn: false) + run(trainer) + } + + override func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer(model: "VQ", bn: false) + run(trainer) + } + + override func testResizeBilinear1() throws + { + let trainer = _buildTrainer(model: "ResizeBilinear1", bn: false) + run(trainer) + } + + override func testResizeBilinear2() throws + { + let trainer = _buildTrainer(model: "ResizeBilinear2", bn: false) + run(trainer) + } + + override func testSelfCorrelate() throws + { + let trainer = _buildTrainer(model: "SelfCorrelate", bn: false) + run(trainer) + } + + override func testNormalize1() throws + { + let trainer = _buildTrainer(model: "Normalize1", bn: false) + run(trainer) + } + + override func testNormalize12() throws + { + let trainer = _buildTrainer(model: "Normalize12", bn: false) + run(trainer) + } + + override func testFlipHorizontal1() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal1", bn: false) + run(trainer) + } + + override func testFlipHorizontal2() throws + { + let trainer = _buildTrainer(model: "FlipHorizontal2", bn: false) + run(trainer) + } + + override func testFlipVertical1() throws + { + let trainer = _buildTrainer(model: "FlipVertical1", bn: false) + run(trainer) + } + + override func testFlipVertical2() throws + { + let trainer = _buildTrainer(model: "FlipVertical2", bn: false) + run(trainer) + } + + override func testLayerOutput() throws + { + let trainer = _buildTrainer(model: "LayerOutput", bn: false) + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class MSE2DGradTests: Input2DMSE2DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer() -> GradTrainer + { + let trainer = GradTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(context: context) + } + return trainer + } + + private func _buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 1, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + _ = try! MSE2D(layerPrev: layer, params: params) + } + + func testLossCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer() + run(trainer) + } + + func testLossGPU() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class MSE2DFlowTests: Input2DMSE2DCase +{ + private func _buildTrainer() -> FlowTrainer + { + let trainer = FlowTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 1, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + _ = try! MSE2D(layerPrev: layer, params: params) + } + + func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class MSE2DFlowResetTests: MSE2DFlowTests +{ + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class MSE2DFlowReverseTests: MSE2DFlowTests +{ + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class MSE2DFlowInferenceTests: MSE2DFlowTests +{ + private func _buildTrainer() -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class MSE2DLoadTests: MSE2DFlowTests +{ + private func _buildTrainer() -> LoadTrainer + { + let trainer = LoadTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class MSE2DTransformTests: MSE2DFlowTests +{ + private func _buildTrainer() -> TransformTrainer + { + let trainer = TransformTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class FTFrequences2DGradTests: FTFrequences2DMSE1DCase +{ + override func setUp() + { + super.setUp() + + optimizerParams.nbLoops = 2 + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer() -> GradTrainer + { + let trainer = GradTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(context: context) + } + return trainer + } + + private func _buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + let frequences: Layer2D = try! FTFrequences2D( + nbChannels: 6, dimension: width, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + layer = try! Multiply2D( + layersPrev: [layer, frequences], params: params + ) + + var head: Layer1D = AvgPool2D(layerPrev: layer, params: params) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: SoftReLU.str, biases: true, params: params + ) + + head = MSE1D(layerPrev: head, params: params) + } + + func testEvenCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer() + run(trainer) + } + + func testEvenGPU() throws + { + let trainer = _buildTrainer() + run(trainer) + } + + func testOddCPU() throws + { + height = 7 + width = 7 + GrAI.Opti.CPU = true + let trainer = _buildTrainer() + run(trainer) + } + + func testOddGPU() throws + { + height = 7 + width = 7 + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class FTFrequences2DFlowTests: FTFrequences2DMSE1DCase +{ + private func _buildTrainer() -> FlowTrainer + { + let trainer = FlowTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + let frequences: Layer2D = try! FTFrequences2D( + nbChannels: 6, dimension: width, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + layer = try! Multiply2D( + layersPrev: [layer, frequences], params: params + ) + + var head: Layer1D = AvgPool2D(layerPrev: layer, params: params) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: LeakyReLU.str, biases: true, params: params + ) + + head = MSE1D(layerPrev: head, params: params) + } + + func testEven() throws + { + let trainer = _buildTrainer() + run(trainer) + } + + func testOdd() throws + { + height = 7 + width = 7 + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class FTFrequences2DFlowResetTests: FTFrequences2DFlowTests +{ + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testEven() throws + { + let trainer = _buildTrainer() + run(trainer) + } + + override func testOdd() throws + { + height = 7 + width = 7 + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class FTFrequences2DFlowReverseTests: FTFrequences2DFlowTests +{ + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testEven() throws + { + let trainer = _buildTrainer() + run(trainer) + } + + override func testOdd() throws + { + height = 7 + width = 7 + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class FTFrequences2DFlowInferenceTests: FTFrequences2DFlowTests +{ + private func _buildTrainer() -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testEven() throws + { + let trainer = _buildTrainer() + run(trainer) + } + + override func testOdd() throws + { + height = 7 + width = 7 + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class FTFrequences2DLoadTests: FTFrequences2DFlowTests +{ + private func _buildTrainer() -> LoadTrainer + { + let trainer = LoadTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testEven() throws + { + let trainer = _buildTrainer() + run(trainer) + } + + override func testOdd() throws + { + height = 7 + width = 7 + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class FTFrequences2DTransformTests: FTFrequences2DFlowTests +{ + private func _buildTrainer() -> TransformTrainer + { + let trainer = TransformTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testEven() throws + { + let trainer = _buildTrainer() + run(trainer) + } + + override func testOdd() throws + { + height = 7 + width = 7 + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-5 and less. +// ----------------------------------------------------------------------------- +class SimilarityBatchError2DGradTests: Input2DSimilarityBatchError2DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer() -> GradTrainer + { + let trainer = GradTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(context: context) + } + return trainer + } + + private func _buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 5, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + layer = SelfCorrelate2D(layerPrev: layer, params: params) + + layer = Normalize122D(layerPrev: layer, params: params) + + _ = try! SimilarityBatchError2D(layerPrev: layer, params: params) + } + + func testCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer() + run(trainer) + } + + func testGPU() throws + { + let trainer = _buildTrainer() + run(trainer, diffThreshold: 0.0001) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-5 and less. +// ----------------------------------------------------------------------------- +class SimilarityBatchError2DFlowTests: Input2DSimilarityBatchError2DCase +{ + private func _buildTrainer() -> FlowTrainer + { + let trainer = FlowTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 5, stride: 1, + activation: ReLU.str, biases: true, bn: false, params: params + ) + + layer = SelfCorrelate2D(layerPrev: layer, params: params) + + layer = Normalize122D(layerPrev: layer, params: params) + + _ = try! SimilarityBatchError2D(layerPrev: layer, params: params) + } + + func test() throws + { + let trainer = _buildTrainer() + run(trainer, diffThreshold: 0.0001) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-5 and less. +// ----------------------------------------------------------------------------- +class SimilarityBatchError2DFlowResetTests: SimilarityBatchError2DFlowTests +{ + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer, nbRetry: 5, diffThreshold: 0.0001) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-5 and less. +// ----------------------------------------------------------------------------- +class SimilarityBatchError2DFlowReverseTests: SimilarityBatchError2DFlowTests +{ + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer, nbRetry: 5, diffThreshold: 0.0001) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SimilarityBatchError2DFlowInferenceTests: SimilarityBatchError2DFlowTests +{ + private func _buildTrainer() -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SimilarityBatchError2DLoadTests: SimilarityBatchError2DFlowTests +{ + private func _buildTrainer() -> LoadTrainer + { + let trainer = LoadTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SimilarityBatchError2DTransformTests: SimilarityBatchError2DFlowTests +{ + private func _buildTrainer() -> TransformTrainer + { + let trainer = TransformTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class SimilarityError2DGradTests: Input2DSimilarityError2DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer() -> GradTrainer + { + let trainer = GradTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(context: context) + } + return trainer + } + + private func _buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + let layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + var layer1, layer2: Layer2D + layer1 = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 5, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + layer1 = SelfCorrelate2D(layerPrev: layer1, params: params) + layer1 = Normalize122D(layerPrev: layer1, params: params) + + layer2 = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 5, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + layer2 = SelfCorrelate2D(layerPrev: layer2, params: params) + layer2 = Normalize122D(layerPrev: layer2, params: params) + + _ = try! SimilarityError2D(layersPrev: [layer1, layer2], params: params) + } + + func testCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer() + run(trainer) + } + + func testGPU() throws + { + let trainer = _buildTrainer() + run(trainer, nbRetry: 5) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class SimilarityError2DFlowTests: Input2DSimilarityError2DCase +{ + private func _buildTrainer() -> FlowTrainer + { + let trainer = FlowTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + let layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + var layer1, layer2: Layer2D + layer1 = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 5, stride: 1, + activation: ReLU.str, biases: true, bn: false, params: params + ) + layer1 = SelfCorrelate2D(layerPrev: layer1, params: params) + layer1 = Normalize122D(layerPrev: layer1, params: params) + + layer2 = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 5, stride: 1, + activation: ReLU.str, biases: true, bn: false, params: params + ) + layer2 = SelfCorrelate2D(layerPrev: layer2, params: params) + layer2 = Normalize122D(layerPrev: layer2, params: params) + + _ = try! SimilarityError2D(layersPrev: [layer1, layer2], params: params) + } + + func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class SimilarityError2DFlowResetTests: SimilarityError2DFlowTests +{ + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class SimilarityError2DFlowReverseTests: SimilarityError2DFlowTests +{ + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SimilarityError2DFlowInferenceTests: SimilarityError2DFlowTests +{ + private func _buildTrainer() -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SimilarityError2DLoadTests: SimilarityError2DFlowTests +{ + private func _buildTrainer() -> LoadTrainer + { + let trainer = LoadTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SimilarityError2DTransformTests: SimilarityError2DFlowTests +{ + private func _buildTrainer() -> TransformTrainer + { + let trainer = TransformTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func test() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCE2DGradTests: Input2DBCE2DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer() -> GradTrainer + { + let trainer = GradTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(context: context) + } + return trainer + } + + private func _buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 1, stride: 1, + activation: Sigmoid.str, biases: true, bn: false, params: params + ) + + _ = try! BCE2D(layerPrev: layer, params: params) + } + + func testLossCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer() + run(trainer) + } + + func testLossGPU() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCE2DFlowTests: Input2DBCE2DCase +{ + private func _buildTrainer() -> FlowTrainer + { + let trainer = FlowTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 1, stride: 1, + activation: Sigmoid.str, biases: true, bn: false, params: params + ) + + _ = try! BCE2D(layerPrev: layer, params: params) + } + + func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCE2DFlowResetTests: BCE2DFlowTests +{ + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCE2DFlowReverseTests: BCE2DFlowTests +{ + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCE2DFlowInferenceTests: BCE2DFlowTests +{ + private func _buildTrainer() -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCE2DLoadTests: BCE2DFlowTests +{ + private func _buildTrainer() -> LoadTrainer + { + let trainer = LoadTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCE2DTransformTests: BCE2DFlowTests +{ + private func _buildTrainer() -> TransformTrainer + { + let trainer = TransformTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid2DGradTests: Input2DBCESigmoid2DCase +{ + override func setUp() + { + super.setUp() + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer() -> GradTrainer + { + let trainer = GradTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(context: context) + } + return trainer + } + + private func _buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 1, stride: 1, + activation: nil, biases: true, bn: false, params: params + ) + + _ = try! BCESigmoid2D(layerPrev: layer, params: params) + } + + func testLossCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer() + run(trainer) + } + + func testLossGPU() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- // Compare GPU gradients with CPU ones through time. // We expect to see errors ~ 1e-7 and less. // ----------------------------------------------------------------------------- -class FTFrequences2DFlowReverseTests: FTFrequences2DFlowTests +class BCESigmoid2DFlowTests: Input2DBCESigmoid2DCase { - private func _buildTrainer() -> FlowReverseTrainer + private func _buildTrainer() -> FlowTrainer { - let trainer = FlowReverseTrainer( + let trainer = FlowTrainer( name: "Layer2D", params: optimizerParams ) @@ -3154,16 +5557,83 @@ class FTFrequences2DFlowReverseTests: FTFrequences2DFlowTests return trainer } - override func testEven() throws + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 1, stride: 1, + activation: nil, biases: true, bn: false, params: params + ) + + _ = try! BCESigmoid2D(layerPrev: layer, params: params) + } + + func testLoss() throws { let trainer = _buildTrainer() run(trainer) } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid2DFlowResetTests: BCESigmoid2DFlowTests +{ + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } - override func testOdd() throws + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid2DFlowReverseTests: BCESigmoid2DFlowTests +{ + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws { - height = 7 - width = 7 let trainer = _buildTrainer() run(trainer) } @@ -3173,7 +5643,7 @@ class FTFrequences2DFlowReverseTests: FTFrequences2DFlowTests // Compare GPU Loss in inference mode with CPU one. // We expect to see errors ~ 1e-3 and less. // ----------------------------------------------------------------------------- -class FTFrequences2DFlowInferenceTests: FTFrequences2DFlowTests +class BCESigmoid2DFlowInferenceTests: BCESigmoid2DFlowTests { private func _buildTrainer() -> InferenceTrainer { @@ -3189,16 +5659,36 @@ class FTFrequences2DFlowInferenceTests: FTFrequences2DFlowTests return trainer } - override func testEven() throws + override func testLoss() throws { let trainer = _buildTrainer() run(trainer) } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class BCESigmoid2DLoadTests: BCESigmoid2DFlowTests +{ + private func _buildTrainer() -> LoadTrainer + { + let trainer = LoadTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } - override func testOdd() throws + override func testLoss() throws { - height = 7 - width = 7 let trainer = _buildTrainer() run(trainer) } @@ -3206,14 +5696,14 @@ class FTFrequences2DFlowInferenceTests: FTFrequences2DFlowTests // ----------------------------------------------------------------------------- // Compare GPU/CPU Losses in inference mode with the one obtained from a -// loaded model. +// transformed model. // We expect to see errors ~ 1e-3 and less. // ----------------------------------------------------------------------------- -class FTFrequences2DLoadTests: FTFrequences2DFlowTests +class BCESigmoid2DTransformTests: BCESigmoid2DFlowTests { - private func _buildTrainer() -> LoadTrainer + private func _buildTrainer() -> TransformTrainer { - let trainer = LoadTrainer( + let trainer = TransformTrainer( name: "Layer2D", params: optimizerParams ) @@ -3225,16 +5715,137 @@ class FTFrequences2DLoadTests: FTFrequences2DFlowTests return trainer } - override func testEven() throws + override func testLoss() throws { let trainer = _buildTrainer() run(trainer) } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class VQ2DFlowTests: Input2DVQ2DCase +{ + private func _buildTrainer() -> FlowTrainer + { + let trainer = FlowTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } - override func testOdd() throws + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 6, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 1, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + _ = VQ2D(layerPrev: layer, K: 5, params: params) + } + + func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class VQ2DFlowResetTests: VQ2DFlowTests +{ + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class VQ2DFlowReverseTests: VQ2DFlowTests +{ + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class VQ2DFlowInferenceTests: VQ2DFlowTests +{ + private func _buildTrainer() -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws { - height = 7 - width = 7 let trainer = _buildTrainer() run(trainer) } @@ -3242,14 +5853,14 @@ class FTFrequences2DLoadTests: FTFrequences2DFlowTests // ----------------------------------------------------------------------------- // Compare GPU/CPU Losses in inference mode with the one obtained from a -// transformed model. +// loaded model. // We expect to see errors ~ 1e-3 and less. // ----------------------------------------------------------------------------- -class FTFrequences2DTransformTests: FTFrequences2DFlowTests +class VQ2DLoadTests: VQ2DFlowTests { - private func _buildTrainer() -> TransformTrainer + private func _buildTrainer() -> LoadTrainer { - let trainer = TransformTrainer( + let trainer = LoadTrainer( name: "Layer2D", params: optimizerParams ) @@ -3261,16 +5872,36 @@ class FTFrequences2DTransformTests: FTFrequences2DFlowTests return trainer } - override func testEven() throws + override func testLoss() throws { let trainer = _buildTrainer() run(trainer) } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class VQ2DTransformTests: VQ2DFlowTests +{ + private func _buildTrainer() -> TransformTrainer + { + let trainer = TransformTrainer( + name: "Layer2D", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } - override func testOdd() throws + override func testLoss() throws { - height = 7 - width = 7 let trainer = _buildTrainer() run(trainer) } diff --git a/Tests/GrAITests/LayerSeqDirtyTests.swift b/Tests/GrAITests/LayerSeqDirtyTests.swift new file mode 100644 index 00000000..50ee983c --- /dev/null +++ b/Tests/GrAITests/LayerSeqDirtyTests.swift @@ -0,0 +1,364 @@ +// +// LayerSeqDirtyTests.swift +// GrAITests +// +// Created by Jean-François Reboud on 05/03/2023. +// + +import GrAIdient +import GrAITestsUtils + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LayerSeqDirtyGradTests: Input2DMSE1DCase +{ + override func setUp() + { + super.setUp() + + optimizerParams.nbLoops = 2 + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer(_ model: String) -> GradTrainer + { + let trainer = GradTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(model: model, context: context) + } + return trainer + } + + private func _buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + let layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + + var firstLayer: LayerSeq = layerSeq + var secondLayer: LayerSeq + + switch model + { + case "FullyConnectedSeq": + secondLayer = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + + case "LayerNorm": + secondLayer = LayerNormSeq( + layerPrev: layerSeq, activation: nil, params: params + ) + + case "Softmax": + secondLayer = try! SoftmaxSeq( + layerPrev: layerSeq, nbHeads: 1, params: params + ) + + default: + fatalError("Unreachable.") + } + + firstLayer = try! SumSeq( + layersPrev: [firstLayer, secondLayer], params: params + ) + + var head: Layer1D = AvgPoolSeq(layerPrev: firstLayer, params: params) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: SoftReLU.str, biases: true, params: params + ) + + _ = MSE1D(layerPrev: head, params: params) + } + + func testFLCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testFLGPU() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testLayerNormSeqCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + func testLayerNormSeqGPU() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + func testSoftmaxSeqCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + func testSoftmaxSeqGPU() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LayerSeqDirtyFlowTests: Input2DMSE1DCase +{ + private func _buildTrainer(_ model: String) -> FlowTrainer + { + let trainer = FlowTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + func buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + let layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + var firstLayer: LayerSeq = layerSeq + var secondLayer: LayerSeq + + switch model + { + case "Sum": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + secondLayer = try! SumSeq( + layersPrev: [firstLayer, otherLayer], + params: params + ) + + case "Concat2": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 3, + activation: LeakyReLU.str, biases: true, params: params + ) + secondLayer = Concat2Seq( + layersPrev: [firstLayer, otherLayer], + params: params + ) + secondLayer = FullyConnectedSeq( + layerPrev: secondLayer, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "FullyConnectedSeq": + secondLayer = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "LayerNorm": + secondLayer = LayerNormSeq( + layerPrev: layerSeq, activation: nil, params: params + ) + + case "QueryQuery": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + secondLayer = try! QuerySeq( + query: layerSeq, key: otherLayer, nbHeads: 1, + params: params + ) + secondLayer = FullyConnectedSeq( + layerPrev: secondLayer, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "QueryKey": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + secondLayer = try! QuerySeq( + query: otherLayer, key: layerSeq, nbHeads: 1, + params: params + ) + secondLayer = FullyConnectedSeq( + layerPrev: secondLayer, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "Softmax": + secondLayer = try! SoftmaxSeq( + layerPrev: layerSeq, nbHeads: 1, params: params + ) + + case "ValueValue": + var otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 2, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + otherLayer = FullyConnectedSeq( + layerPrev: otherLayer, nbNeurons: 9, + activation: LeakyReLU.str, biases: true, params: params + ) + secondLayer = try! ValueSeq( + value: layerSeq, score: otherLayer, nbHeads: 1, + params: params + ) + + case "ValueScore": + secondLayer = firstLayer + firstLayer = FullyConnectedSeq( + layerPrev: firstLayer, nbNeurons: 9, + activation: LeakyReLU.str, biases: true, params: params + ) + secondLayer = try! ValueSeq( + value: secondLayer, score: firstLayer, nbHeads: 1, + params: params + ) + secondLayer = FullyConnectedSeq( + layerPrev: secondLayer, nbNeurons: 9, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "VQ": + secondLayer = VQSeq(layerPrev: layerSeq, K: 5, params: params) + (secondLayer as! VQSeq).beta = 0.25 + + default: + fatalError("Unreachable.") + } + + firstLayer = try! SumSeq( + layersPrev: [firstLayer, secondLayer], params: params + ) + + var head: Layer1D = AvgPoolSeq(layerPrev: firstLayer, params: params) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: LeakyReLU.str, biases: true, params: params + ) + + _ = MSE1D(layerPrev: head, params: params) + } + + func testSum() throws + { + let trainer = _buildTrainer("Sum") + run(trainer) + } + + func testConcat2() throws + { + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + func testFLSeq() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testLayerNormSeq() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + func testQueryQuerySeq() throws + { + let trainer = _buildTrainer("QueryQuery") + run(trainer) + } + + func testQueryKeySeq() throws + { + let trainer = _buildTrainer("QueryKey") + run(trainer) + } + + func testSoftmaxSeq() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + func testValueValueSeq() throws + { + let trainer = _buildTrainer("ValueValue") + run(trainer) + } + + func testValueScoreSeq() throws + { + let trainer = _buildTrainer("ValueScore") + run(trainer) + } + + func testVQ() throws + { + let trainer = _buildTrainer("VQ") + run(trainer) + } + + func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("VQ") + run(trainer) + } +} diff --git a/Tests/GrAITests/LayerSeqTests.swift b/Tests/GrAITests/LayerSeqTests.swift new file mode 100644 index 00000000..3e60c066 --- /dev/null +++ b/Tests/GrAITests/LayerSeqTests.swift @@ -0,0 +1,2048 @@ +// +// LayerSeqTests.swift +// GrAITests +// +// Created by Jean-François Reboud on 27/02/2023. +// + +import XCTest +import GrAIdient +import GrAITestsUtils + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LayerSeqGradTests: Input2DMSE1DCase +{ + override func setUp() + { + super.setUp() + + optimizerParams.nbLoops = 2 + GrAI.Loop.gradientChecking = true + } + + private func _buildTrainer(_ model: String) -> GradTrainer + { + let trainer = GradTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(model: model, context: context) + } + return trainer + } + + private func _buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + var layerSeq: LayerSeq + switch model + { + case "FullyConnectedPatch": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + + case "Sum": + let otherLayer1: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + let otherLayer2: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! SumSeq( + layersPrev: [layerSeq, otherLayer1, otherLayer2], + params: params + ) + + case "Concat1": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 2, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! Concat1Seq( + layersPrev: [layerSeq, otherLayer], + params: params + ) + + case "Concat2": + let otherLayer1: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 3, + activation: SoftReLU.str, biases: true, params: params + ) + let otherLayer2: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 6, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 9, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = Concat2Seq( + layersPrev: [layerSeq, otherLayer1, otherLayer2], + params: params + ) + + case "Constant12": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 3, nbNeurons: 2, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = Constant12Seq( + sequence: 4, nbNeurons: 2, params: params + ) + (layerSeq as! Constant12Seq).weightsCPU = [ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0 + ] + + layerSeq = try! SumSeq( + layersPrev: [layerSeq, otherLayer], params: params + ) + + case "Constant2": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 2, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = Constant2Seq( + sequence: 9, nbNeurons: 5, params: params + ) + (layerSeq as! Constant2Seq).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] + + layerSeq = try! SumSeq( + layersPrev: [layerSeq, otherLayer], params: params + ) + + case "FullyConnectedSeq": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 4, + activation: SoftReLU.str, biases: true, params: params + ) + + case "LayerNorm": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = LayerNormSeq( + layerPrev: layerSeq, activation: nil, params: params + ) + + case "Query": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 6, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 6, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! QuerySeq( + query: layerSeq, key: otherLayer, nbHeads: 2, params: params + ) + + case "Softmax": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 15, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! SoftmaxSeq( + layerPrev: layerSeq, nbHeads: 3, params: params + ) + + case "Value": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 3, nbNeurons: 6, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 3, nbNeurons: 6, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 2 * 4, + activation: SoftReLU.str, biases: true, params: params + ) + layerSeq = try! ValueSeq( + value: otherLayer, score: layerSeq, nbHeads: 2, params: params + ) + + default: + fatalError("Unreachable.") + } + + var head: Layer1D = AvgPoolSeq(layerPrev: layerSeq, params: params) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: SoftReLU.str, biases: true, params: params + ) + + _ = MSE1D(layerPrev: head, params: params) + } + + func testFullyConnectedPatchCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + func testFullyConnectedPatchGPU() throws + { + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + func testFullyConnectedPatchSampleGPU() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + func testSumCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Sum") + run(trainer) + } + + func testSumGPU() throws + { + let trainer = _buildTrainer("Sum") + run(trainer) + } + + func testConcat1CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Concat1") + run(trainer) + } + + func testConcat1GPU() throws + { + let trainer = _buildTrainer("Concat1") + run(trainer) + } + + func testConcat2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + func testConcat2GPU() throws + { + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + func testConstant12CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + func testConstant12GPU() throws + { + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + func testConstant2CPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + func testConstant2GPU() throws + { + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + func testConstant2SampleGPU() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + func testFullyConnectedSeqCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testFullyConnectedSeqGPU() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testFullyConnectedSeqSampleGPU() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testLayerNormSeqCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + func testLayerNormSeqGPU() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + func testQuerySeqCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Query") + run(trainer) + } + + func testQuerySeqGPU() throws + { + let trainer = _buildTrainer("Query") + run(trainer) + } + + func testSoftmaxSeqCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + func testSoftmaxSeqGPU() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + func testValueSeqCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer("Value") + run(trainer) + } + + func testValueSeqGPU() throws + { + let trainer = _buildTrainer("Value") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LayerSeqFlowTests: Input2DMSE1DCase +{ + private func _buildTrainer(_ model: String) -> FlowTrainer + { + let trainer = FlowTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + func buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + var layerSeq: LayerSeq + switch model + { + case "FullyConnectedPatch": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "Sum": + let otherLayer1: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + let otherLayer2: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! SumSeq( + layersPrev: [layerSeq, otherLayer1, otherLayer2], + params: params + ) + + case "Concat1": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 2, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! Concat1Seq( + layersPrev: [layerSeq, otherLayer], + params: params + ) + + case "Concat2": + let otherLayer1: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 3, + activation: LeakyReLU.str, biases: true, params: params + ) + let otherLayer2: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 6, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 9, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = Concat2Seq( + layersPrev: [layerSeq, otherLayer1, otherLayer2], + params: params + ) + + case "Constant12": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 3, nbNeurons: 2, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = Constant12Seq( + sequence: 4, nbNeurons: 2, params: params + ) + (layerSeq as! Constant12Seq).weightsCPU = [ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0 + ] + + layerSeq = try! SumSeq( + layersPrev: [layerSeq, otherLayer], params: params + ) + + case "Constant2": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 2, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = Constant2Seq( + sequence: 9, nbNeurons: 5, params: params + ) + (layerSeq as! Constant2Seq).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] + + layerSeq = try! SumSeq( + layersPrev: [layerSeq, otherLayer], params: params + ) + + case "FullyConnectedSeq": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 4, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "LayerNorm": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = LayerNormSeq( + layerPrev: layerSeq, activation: nil, params: params + ) + + case "Query": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 6, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 6, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! QuerySeq( + query: layerSeq, key: otherLayer, nbHeads: 2, params: params + ) + + case "Softmax": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 15, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! SoftmaxSeq( + layerPrev: layerSeq, nbHeads: 3, params: params + ) + + case "Value": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 3, nbNeurons: 6, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 3, nbNeurons: 6, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 2 * 4, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = try! ValueSeq( + value: otherLayer, score: layerSeq, nbHeads: 2, params: params + ) + + case "VQ": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = VQSeq(layerPrev: layerSeq, K: 5, params: params) + (layerSeq as! VQSeq).beta = 0.25 + + default: + fatalError("Unreachable.") + } + + var head: Layer1D = AvgPoolSeq(layerPrev: layerSeq, params: params) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: LeakyReLU.str, biases: true, params: params + ) + + _ = MSE1D(layerPrev: head, params: params) + } + + func testFullyConnectedPatch() throws + { + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + func testFullyConnectedPatchSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + func testSum() throws + { + let trainer = _buildTrainer("Sum") + run(trainer) + } + + func testConcat1() throws + { + let trainer = _buildTrainer("Concat1") + run(trainer) + } + + func testConcat2() throws + { + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + func testConstant12() throws + { + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + func testConstant2() throws + { + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + func testConstant2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + func testFullyConnectedSeq() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testFullyConnectedSeqSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testLayerNormSeq() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + func testQuerySeq() throws + { + let trainer = _buildTrainer("Query") + run(trainer) + } + + func testSoftmaxSeq() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + func testValueSeq() throws + { + let trainer = _buildTrainer("Value") + run(trainer) + } + + func testVQ() throws + { + let trainer = _buildTrainer("VQ") + run(trainer) + } + + func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("VQ") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LayerSeqFlowResetTests: LayerSeqFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(_ model: String) -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFullyConnectedPatch() throws + { + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testFullyConnectedPatchSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testSum() throws + { + let trainer = _buildTrainer("Sum") + run(trainer) + } + + override func testConcat1() throws + { + let trainer = _buildTrainer("Concat1") + run(trainer) + } + + override func testConcat2() throws + { + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + override func testConstant12() throws + { + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + override func testConstant2() throws + { + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testConstant2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testFullyConnectedSeq() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testFullyConnectedSeqSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testLayerNormSeq() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + override func testQuerySeq() throws + { + let trainer = _buildTrainer("Query") + run(trainer) + } + + override func testSoftmaxSeq() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + override func testValueSeq() throws + { + let trainer = _buildTrainer("Value") + run(trainer) + } + + override func testVQ() throws + { + let trainer = _buildTrainer("VQ") + run(trainer) + } + + override func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("VQ") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LayerSeqFlowReverseTests: LayerSeqFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer(_ model: String) -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFullyConnectedPatch() throws + { + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testFullyConnectedPatchSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testSum() throws + { + let trainer = _buildTrainer("Sum") + run(trainer) + } + + override func testConcat1() throws + { + let trainer = _buildTrainer("Concat1") + run(trainer) + } + + override func testConcat2() throws + { + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + override func testConstant12() throws + { + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + override func testConstant2() throws + { + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testConstant2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testFullyConnectedSeq() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testFullyConnectedSeqSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testLayerNormSeq() throws + { + /*let trainer = _buildTrainer("LayerNorm") + run(trainer, nbRetry: 5)*/ + } + + override func testQuerySeq() throws + { + let trainer = _buildTrainer("Query") + run(trainer) + } + + override func testSoftmaxSeq() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + override func testValueSeq() throws + { + let trainer = _buildTrainer("Value") + run(trainer) + } + + override func testVQ() throws + { + let trainer = _buildTrainer("VQ") + run(trainer) + } + + override func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("VQ") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class LayerSeqFlowAccumulateTests: Input2DMSE1DCase +{ + private func _buildTrainer(_ model: String) -> FlowTrainer + { + let trainer = FlowAccumulateTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + func buildModel(model: String, context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + var layerSeq: LayerSeq + switch model + { + case "FullyConnectedPatch": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "Constant12": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 3, nbNeurons: 2, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = Constant12Seq( + sequence: 4, nbNeurons: 2, params: params + ) + (layerSeq as! Constant12Seq).weightsCPU = [ + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0 + ] + + layerSeq = try! SumSeq( + layersPrev: [layerSeq, otherLayer], params: params + ) + + case "Constant2": + let otherLayer: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: 2, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = Constant2Seq( + sequence: 9, nbNeurons: 5, params: params + ) + (layerSeq as! Constant2Seq).weightsCPU = [1.0, 2.0, 3.0, 4.0, 5.0] + + layerSeq = try! SumSeq( + layersPrev: [layerSeq, otherLayer], params: params + ) + + case "FullyConnectedSeq": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 4, + activation: LeakyReLU.str, biases: true, params: params + ) + + case "LayerNorm": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = LayerNormSeq( + layerPrev: layerSeq, activation: nil, params: params + ) + + case "VQ": + layerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + layerSeq = VQSeq(layerPrev: layerSeq, K: 5, params: params) + (layerSeq as! VQSeq).beta = 0.25 + + default: + fatalError("Unreachable.") + } + + var head: Layer1D = AvgPoolSeq(layerPrev: layerSeq, params: params) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: LeakyReLU.str, biases: true, params: params + ) + + _ = MSE1D(layerPrev: head, params: params) + } + + func testFullyConnectedPatch() throws + { + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + func testFullyConnectedPatchSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + func testConstant12() throws + { + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + func testConstant2() throws + { + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + func testConstant2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + func testFullyConnectedSeq() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testFullyConnectedSeqSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + func testLayerNormSeq() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + func testVQ() throws + { + let trainer = _buildTrainer("VQ") + run(trainer) + } + + func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("VQ") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class LayerSeqInferenceTests: LayerSeqFlowTests +{ + private func _buildTrainer(_ model: String) -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFullyConnectedPatch() throws + { + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testFullyConnectedPatchSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testSum() throws + { + let trainer = _buildTrainer("Sum") + run(trainer) + } + + override func testConcat1() throws + { + let trainer = _buildTrainer("Concat1") + run(trainer) + } + + override func testConcat2() throws + { + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + override func testConstant12() throws + { + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + override func testConstant2() throws + { + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testConstant2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testFullyConnectedSeq() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testFullyConnectedSeqSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testLayerNormSeq() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + override func testQuerySeq() throws + { + let trainer = _buildTrainer("Query") + run(trainer) + } + + override func testSoftmaxSeq() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + override func testValueSeq() throws + { + let trainer = _buildTrainer("Value") + run(trainer) + } + + override func testVQ() throws + { + let trainer = _buildTrainer("VQ") + run(trainer) + } + + override func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("VQ") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class LayerSeqLoadTests: LayerSeqFlowTests +{ + private func _buildTrainer(_ model: String) -> LoadTrainer + { + let trainer = LoadTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFullyConnectedPatch() throws + { + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testFullyConnectedPatchSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testSum() throws + { + let trainer = _buildTrainer("Sum") + run(trainer) + } + + override func testConcat1() throws + { + let trainer = _buildTrainer("Concat1") + run(trainer) + } + + override func testConcat2() throws + { + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + override func testConstant12() throws + { + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + override func testConstant2() throws + { + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testConstant2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testFullyConnectedSeq() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testFullyConnectedSeqSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testLayerNormSeq() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + override func testQuerySeq() throws + { + let trainer = _buildTrainer("Query") + run(trainer) + } + + override func testSoftmaxSeq() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + override func testValueSeq() throws + { + let trainer = _buildTrainer("Value") + run(trainer) + } + + override func testVQ() throws + { + let trainer = _buildTrainer("VQ") + run(trainer) + } + + override func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("VQ") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class LayerSeqTransformTests: LayerSeqFlowTests +{ + /// + /// Run Transform tests. + /// + /// The goal is to compare the losses computed in the CPU execution + /// after transforming the model and do the same in the GPU execution context. + /// + /// - Parameters: + /// - trainer: The testing pipeline to run. + /// - nbRetry: The maximum number we can retry the test. + /// - diffThreshold: The threshold above which the relative difference is too high. + /// + func run( + _ trainer: TransformTrainer, + nbRetry: Int = NB_RETRY, + diffThreshold: Double = 0.001) + { + retryNumeric( + nbRetry: nbRetry, + { + () throws in + try trainer.run( + transforms: [self.copy, self.copyInPlace], + setData: self.setData, + setLoss: self.setLoss, + getLoss: self.getLoss) + { + (diffCPU: Double, diffGPU: Double) in + if diffCPU > diffThreshold + { + throw TestError.Numeric + } + if diffGPU > diffThreshold + { + throw TestError.Numeric + } + } + }, + { + () in + XCTAssert(false) + } + ) + } + + private func _buildTrainer(_ model: String) -> TransformTrainer + { + let trainer = TransformTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(model: model, context: context) + } + return trainer + } + + override func testFullyConnectedPatch() throws + { + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testFullyConnectedPatchSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedPatch") + run(trainer) + } + + override func testSum() throws + { + let trainer = _buildTrainer("Sum") + run(trainer) + } + + override func testConcat1() throws + { + let trainer = _buildTrainer("Concat1") + run(trainer) + } + + override func testConcat2() throws + { + let trainer = _buildTrainer("Concat2") + run(trainer) + } + + override func testConstant12() throws + { + let trainer = _buildTrainer("Constant12") + run(trainer) + } + + override func testConstant2() throws + { + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testConstant2Sample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("Constant2") + run(trainer) + } + + override func testFullyConnectedSeq() throws + { + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testFullyConnectedSeqSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("FullyConnectedSeq") + run(trainer) + } + + override func testLayerNormSeq() throws + { + let trainer = _buildTrainer("LayerNorm") + run(trainer) + } + + override func testQuerySeq() throws + { + let trainer = _buildTrainer("Query") + run(trainer) + } + + override func testSoftmaxSeq() throws + { + let trainer = _buildTrainer("Softmax") + run(trainer) + } + + override func testValueSeq() throws + { + let trainer = _buildTrainer("Value") + run(trainer) + } + + override func testVQ() throws + { + let trainer = _buildTrainer("VQ") + run(trainer) + } + + override func testVQSample() throws + { + GrAI.Gradient.sample = true + let trainer = _buildTrainer("VQ") + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Gradient Checking +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class SelectSeqGradTests: Input2DMSE1DCase +{ + private func _buildTrainer() -> GradTrainer + { + let trainer = GradTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + _buildModel(context: context) + } + return trainer + } + + private func _buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: SoftReLU.str, biases: true, bn: false, params: params + ) + + let layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: SoftReLU.str, biases: true, params: params + ) + + var head: Layer1D = SelectSeq( + layerPrev: layerSeq, + targetSeq: 3, + params: params + ) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: SoftReLU.str, biases: true, params: params + ) + + _ = MSE1D(layerPrev: head, params: params) + } + + func testSelectCPU() throws + { + GrAI.Opti.CPU = true + let trainer = _buildTrainer() + run(trainer) + } + + func testSelectGPU() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class SelectSeqFlowTests: Input2DMSE1DCase +{ + private func _buildTrainer() -> FlowTrainer + { + let trainer = FlowTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + let layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + var head: Layer1D = SelectSeq( + layerPrev: layerSeq, + targetSeq: 3, + params: params + ) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: LeakyReLU.str, biases: true, params: params + ) + + _ = MSE1D(layerPrev: head, params: params) + } + + func testSelect() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class SelectSeqFlowResetTests: SelectSeqFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testSelect() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class SelectSeqFlowReverseTests: SelectSeqFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testSelect() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SelectSeqInferenceTests: SelectSeqFlowTests +{ + private func _buildTrainer() -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testSelect() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SelectSeqLoadTests: SelectSeqFlowTests +{ + private func _buildTrainer() -> LoadTrainer + { + let trainer = LoadTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testSelect() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class SelectSeqTransformTests: SelectSeqFlowTests +{ + /// + /// Run Transform tests. + /// + /// The goal is to compare the losses computed in the CPU execution + /// after transforming the model and do the same in the GPU execution context. + /// + /// - Parameters: + /// - trainer: The testing pipeline to run. + /// - nbRetry: The maximum number we can retry the test. + /// - diffThreshold: The threshold above which the relative difference is too high. + /// + func run( + _ trainer: TransformTrainer, + nbRetry: Int = NB_RETRY, + diffThreshold: Double = 0.001) + { + retryNumeric( + nbRetry: nbRetry, + { + () throws in + try trainer.run( + transforms: [self.copy, self.copyInPlace], + setData: self.setData, + setLoss: self.setLoss, + getLoss: self.getLoss) + { + (diffCPU: Double, diffGPU: Double) in + if diffCPU > diffThreshold + { + throw TestError.Numeric + } + if diffGPU > diffThreshold + { + throw TestError.Numeric + } + } + }, + { + () in + XCTAssert(false) + } + ) + } + + private func _buildTrainer() -> TransformTrainer + { + let trainer = TransformTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testSelect() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class VQSeqFlowTests: Input2DVQSeqCase +{ + private func _buildTrainer() -> FlowTrainer + { + let trainer = FlowTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + func buildModel(context: ModelContext) + { + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = Input2D( + nbChannels: 1, width: width, height: height, params: params + ) + + layer = Convolution2D( + layerPrev: layer, size: 1, nbChannels: 3, stride: 1, + activation: LeakyReLU.str, biases: true, bn: false, params: params + ) + + let layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: width / 3, nbNeurons: 5, + activation: LeakyReLU.str, biases: true, params: params + ) + + _ = VQSeq(layerPrev: layerSeq, K: 5, params: params) + } + + func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class VQSeqFlowResetTests: VQSeqFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer() -> FlowResetTrainer + { + let trainer = FlowResetTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU gradients with CPU ones through time. +// We expect to see errors ~ 1e-7 and less. +// ----------------------------------------------------------------------------- +class VQSeqFlowReverseTests: VQSeqFlowTests +{ + override func setUp() + { + super.setUp() + + setOptimizerParams(params: &optimizerParams, + optimizerClass: .Adam) + } + + private func _buildTrainer() -> FlowReverseTrainer + { + let trainer = FlowReverseTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU Loss in inference mode with CPU one. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class VQSeqInferenceTests: VQSeqFlowTests +{ + private func _buildTrainer() -> InferenceTrainer + { + let trainer = InferenceTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// loaded model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class VQSeqLoadTests: VQSeqFlowTests +{ + private func _buildTrainer() -> LoadTrainer + { + let trainer = LoadTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} + +// ----------------------------------------------------------------------------- +// Compare GPU/CPU Losses in inference mode with the one obtained from a +// transformed model. +// We expect to see errors ~ 1e-3 and less. +// ----------------------------------------------------------------------------- +class VQSeqTransformTests: VQSeqFlowTests +{ + private func _buildTrainer() -> TransformTrainer + { + let trainer = TransformTrainer( + name: "LayerSeq", + params: optimizerParams + ) + trainer.build() + { + (context: ModelContext) in + buildModel(context: context) + } + return trainer + } + + override func testLoss() throws + { + let trainer = _buildTrainer() + run(trainer) + } +} diff --git a/Tests/GrAITests/LinearError1DTests.swift b/Tests/GrAITests/LinearError1DTests.swift deleted file mode 100644 index dfc03c3d..00000000 --- a/Tests/GrAITests/LinearError1DTests.swift +++ /dev/null @@ -1,409 +0,0 @@ -// -// LinearError1DTests.swift -// GrAITests -// -// Created by Jean-François Reboud on 10/10/2022. -// - -import GrAIdient -import GrAITestsUtils - -// ----------------------------------------------------------------------------- -// Gradient Checking -// We expect to see errors ~ 1e-7 and less. -// ----------------------------------------------------------------------------- -class LinearError1DGradTests: Input1DLinearError1DCase -{ - override func setUp() - { - super.setUp() - GrAI.Loop.gradientChecking = true - } - - private func _buildTrainer(_ model: String) -> GradTrainer - { - let trainer = GradTrainer( - name: "LinearError1D", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - _buildModel(model: model, context: context) - } - return trainer - } - - private func _buildModel(model: String, context: ModelContext) - { - let params = GrAI.Model.Params(context: context) - - var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - - layer = FullyConnected( - layerPrev: layer, nbNeurons: 5, - activation: SoftReLU.str, biases: true, - params: params - ) - - switch model - { - case "FullyConnected": - layer = FullyConnected( - layerPrev: layer, nbNeurons: 12, - activation: SoftReLU.str, biases: true, - params: params - ) - - case "Activation": - layer = Activation1D( - layerPrev: layer, - activation: SoftReLU.str, - params: params - ) - - default: - fatalError("Unreachable.") - } - - layer = FullyConnected( - layerPrev: layer, nbNeurons: 1, - activation: SoftReLU.str, biases: true, - params: params - ) - - layer = LinearError1D(layerPrev: layer, params: params) - } - - func testFLCPU() throws - { - GrAI.Opti.CPU = true - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - func testFLGPU() throws - { - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - func testFLSampleGPU() throws - { - GrAI.Gradient.sample = true - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - func testActivationCPU() throws - { - GrAI.Opti.CPU = true - let trainer = _buildTrainer("Activation") - run(trainer) - } - - func testActivationGPU() throws - { - let trainer = _buildTrainer("Activation") - run(trainer) - } -} - -// ----------------------------------------------------------------------------- -// Compare GPU gradients with CPU ones through time. -// We expect to see errors ~ 1e-7 and less. -// ----------------------------------------------------------------------------- -class LinearError1DFlowTests: Input1DLinearError1DCase -{ - private func _buildTrainer(_ model: String) -> FlowTrainer - { - let trainer = FlowTrainer( - name: "LinearError", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(model: model, context: context) - } - return trainer - } - - func buildModel(model: String, context: ModelContext) - { - let params = GrAI.Model.Params(context: context) - - var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - - layer = FullyConnected( - layerPrev: layer, nbNeurons: 5, - activation: LeakyReLU.str, biases: true, - params: params - ) - - switch model - { - case "FullyConnected": - layer = FullyConnected( - layerPrev: layer, nbNeurons: 12, - activation: LeakyReLU.str, biases: true, - params: params - ) - - case "Activation": - layer = Activation1D( - layerPrev: layer, - activation: LeakyReLU.str, - params: params - ) - - default: - fatalError("Unreachable.") - } - - layer = FullyConnected( - layerPrev: layer, nbNeurons: 1, - activation: LeakyReLU.str, biases: true, - params: params - ) - - layer = LinearError1D(layerPrev: layer, params: params) - } - - func testFL() throws - { - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - func testFLSample() throws - { - GrAI.Gradient.sample = true - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - func testActivation() throws - { - let trainer = _buildTrainer("Activation") - run(trainer) - } -} - -// ----------------------------------------------------------------------------- -// Compare GPU Loss in inference mode with CPU one. -// We expect to see errors ~ 1e-7 and less. -// ----------------------------------------------------------------------------- -class LinearError1DFlowResetTests: LinearError1DFlowTests -{ - override func setUp() - { - super.setUp() - - setOptimizerParams(params: &optimizerParams, - optimizerClass: .Adam) - } - - private func _buildTrainer(_ model: String) -> FlowResetTrainer - { - let trainer = FlowResetTrainer( - name: "LinearError", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(model: model, context: context) - } - return trainer - } - - override func testFL() throws - { - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testFLSample() throws - { - GrAI.Gradient.sample = true - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testActivation() throws - { - let trainer = _buildTrainer("Activation") - run(trainer) - } -} - -// ----------------------------------------------------------------------------- -// Compare GPU Loss in inference mode with CPU one. -// We expect to see errors ~ 1e-7 and less. -// ----------------------------------------------------------------------------- -class LinearError1DFlowReverseTests: LinearError1DFlowTests -{ - override func setUp() - { - super.setUp() - - setOptimizerParams(params: &optimizerParams, - optimizerClass: .Adam) - } - - private func _buildTrainer(_ model: String) -> FlowReverseTrainer - { - let trainer = FlowReverseTrainer( - name: "LinearError", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(model: model, context: context) - } - return trainer - } - - override func testFL() throws - { - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testFLSample() throws - { - GrAI.Gradient.sample = true - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testActivation() throws - { - let trainer = _buildTrainer("Activation") - run(trainer) - } -} - -// ----------------------------------------------------------------------------- -// Compare GPU Loss in inference mode with CPU one. -// We expect to see errors ~ 1e-3 and less. -// ----------------------------------------------------------------------------- -class LinearError1DInferenceTests: LinearError1DFlowTests -{ - private func _buildTrainer(_ model: String) -> InferenceTrainer - { - let trainer = InferenceTrainer( - name: "LinearError", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(model: model, context: context) - } - return trainer - } - - override func testFL() throws - { - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testFLSample() throws - { - GrAI.Gradient.sample = true - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testActivation() throws - { - let trainer = _buildTrainer("Activation") - run(trainer) - } -} - -// ----------------------------------------------------------------------------- -// Compare GPU/CPU Losses in inference mode with the one obtained from a -// loaded model. -// We expect to see errors ~ 1e-3 and less. -// ----------------------------------------------------------------------------- -class LinearError1DLoadTests: LinearError1DFlowTests -{ - private func _buildTrainer(_ model: String) -> LoadTrainer - { - let trainer = LoadTrainer( - name: "LinearError", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(model: model, context: context) - } - return trainer - } - - override func testFL() throws - { - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testFLSample() throws - { - GrAI.Gradient.sample = true - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testActivation() throws - { - let trainer = _buildTrainer("Activation") - run(trainer) - } -} - -// ----------------------------------------------------------------------------- -// Compare GPU/CPU Losses in inference mode with the one obtained from a -// transformed model. -// We expect to see errors ~ 1e-3 and less. -// ----------------------------------------------------------------------------- -class LinearError1DTransformTests: LinearError1DFlowTests -{ - private func _buildTrainer(_ model: String) -> TransformTrainer - { - let trainer = TransformTrainer( - name: "LinearError", - params: optimizerParams - ) - trainer.build() - { - (context: ModelContext) in - buildModel(model: model, context: context) - } - return trainer - } - - override func testFL() throws - { - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testFLSample() throws - { - GrAI.Gradient.sample = true - let trainer = _buildTrainer("FullyConnected") - run(trainer) - } - - override func testActivation() throws - { - let trainer = _buildTrainer("Activation") - run(trainer) - } -} diff --git a/Tests/GrAITests/OptimizerTests.swift b/Tests/GrAITests/OptimizerTests.swift index 476a04e6..88c29e10 100644 --- a/Tests/GrAITests/OptimizerTests.swift +++ b/Tests/GrAITests/OptimizerTests.swift @@ -44,19 +44,19 @@ class OptimizerTests: Input1DMSE1DCase var layer: Layer1D = Input1D(nbNeurons: 1, params: params) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: LeakyReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 12, activation: LeakyReLU.str, biases: true, params: params ) - layer = FullyConnected( + layer = try! FullyConnected( layerPrev: layer, nbNeurons: 1, activation: LeakyReLU.str, biases: true, params: params diff --git a/Tests/GrAITests/ReduceTests.swift b/Tests/GrAITests/ReduceTests.swift new file mode 100644 index 00000000..a74092e3 --- /dev/null +++ b/Tests/GrAITests/ReduceTests.swift @@ -0,0 +1,96 @@ +// +// ReduceTests.swift +// GrAITests +// +// Created by Jean-François Reboud on 17/05/2023. +// + +import XCTest +import GrAIdient + +/// Test reduce kernel. +class ReduceTests: XCTestCase +{ + var _buffer: MetalSharedBuffer! = nil + var _array = [Float]() + + override func setUp() + { + _ = MetalKernel.get + } + + private func _testBuffer(dim1: Int, dim2: Int) + { + _array = [Float](repeating: 0.0, count: dim1 * dim2) + _buffer = MetalSharedBuffer(dim1 * dim2, deviceID: 0) + let buffer = _buffer.buffer + + for elem1 in 0.. Model { - let context = ModelContext(name: "ModelTest1", curID: 0) + let context = ModelContext(name: "ModelTestConv1", curID: 0) let params = GrAI.Model.Params(context: context) var layer: Layer2D @@ -55,7 +55,7 @@ class ModelTest1 layer = AdaptiveAvgPool2D(layerPrev: layer, size: 7, params: params) - var head: Layer1D = FullyConnected( + var head: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 10, activation: ReLU.str, @@ -63,7 +63,7 @@ class ModelTest1 params: params ) - head = FullyConnected( + head = try! FullyConnected( layerPrev: head, nbNeurons: 1, activation: nil, @@ -75,7 +75,7 @@ class ModelTest1 // Load weights from `PyTorch`. let pythonLib = Python.import("python_lib") - let data = pythonLib.load_test1_weights() + let data = pythonLib.load_conv1_weights() let weights = [[Float]](data.tuple2.0)! @@ -110,7 +110,7 @@ class ModelTest1 } /// Model to test against PyTorch. -class ModelTest2 +class ModelTestConv2 { /// /// Create the model and import weights from PyTorch. @@ -125,7 +125,7 @@ class ModelTest2 /// static func build(_ size: Int) -> Model { - let context = ModelContext(name: "ModelTest2", curID: 0) + let context = ModelContext(name: "ModelTestConv2", curID: 0) let params = GrAI.Model.Params(context: context) var layer: Layer2D @@ -157,11 +157,11 @@ class ModelTest2 params: params ) - layer = Sum2D(layersPrev: [layer, layer1], params: params) + layer = try! Sum2D(layersPrev: [layer, layer1], params: params) layer = AdaptiveAvgPool2D(layerPrev: layer, size: 7, params: params) - var head: Layer1D = FullyConnected( + var head: Layer1D = try! FullyConnected( layerPrev: layer, nbNeurons: 10, activation: ReLU.str, @@ -169,7 +169,7 @@ class ModelTest2 params: params ) - head = FullyConnected( + head = try! FullyConnected( layerPrev: head, nbNeurons: 1, activation: nil, @@ -181,7 +181,7 @@ class ModelTest2 // Load weights from `PyTorch`. let pythonLib = Python.import("python_lib") - let data = pythonLib.load_test2_weights() + let data = pythonLib.load_conv2_weights() let weights = [[Float]](data.tuple2.0)! @@ -225,7 +225,7 @@ class ModelTest2 } /// Model to test against PyTorch. -class ModelTest3 +class ModelTestFFT { /// /// Create the model. @@ -240,7 +240,7 @@ class ModelTest3 /// static func build(_ size: Int) -> Model { - let context = ModelContext(name: "ModelTest2", curID: 0) + let context = ModelContext(name: "ModelTestFFT", curID: 0) let params = GrAI.Model.Params(context: context) var layer: Layer2D @@ -250,16 +250,16 @@ class ModelTest3 ) let firstLayer: Layer2D = layer - layer = FTFrequences2D( + layer = try! FTFrequences2D( nbChannels: 6, dimension: size, params: params ) - layer = Multiply2D( + layer = try! Multiply2D( layersPrev: [firstLayer, layer], params: params ) - layer = IRDFT2RGB( + layer = try! IRDFT2RGB( layerPrev: layer, params: params ) @@ -269,7 +269,7 @@ class ModelTest3 params: params ) - layer = DecorrelateRGB( + layer = try! DecorrelateRGB( layerPrev: layer, correlation: [ 0.26, 0.09, 0.02, @@ -304,43 +304,474 @@ class ModelTest3 } } +/// Generic model to test against PyTorch. +class ModelTestConv +{ + /// + /// Load weights in the model. + /// + /// - Parameters: + /// - model: The model. + /// - weights: The weights. + /// + static func initWeights(model: Model, weights: [[Float]]) + { + // Apply weights on the `GrAIdient` model's layers. + var cur = 0 + for num_layer in 0.. Model + { + let context = ModelContext(name: "ModelTestConvSK", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: size, + height: size, + params: params + ) + + layer = Convolution2D( + layerPrev: layer, + size: kernel, nbChannels: 5, stride: stride, + activation: nil, biases: true, bn: false, + params: params + ) + + var head: Layer1D = AvgPool2D( + layerPrev: layer, params: params + ) + + head = try! FullyConnected( + layerPrev: head, + nbNeurons: 1, + activation: nil, + biases: true, + params: params + ) + + let model = Model(model: context.model, modelsPrev: []) + + // Load weights from `PyTorch`. + let pythonLib = Python.import("python_lib") + let data = pythonLib.load_conv_sk_weights(stride, kernel) + + let weights = [[Float]](data.tuple2.0)! + super.initWeights(model: model, weights: weights) + + return model + } +} + +/// Model to test against PyTorch. +class ModelTestDeConvSK: ModelTestConv +{ + /// + /// Create the model and import weights from PyTorch. + /// + /// Principle features: + /// - Deconvolution + /// + /// - Parameters: + /// - size: The size of the input data. + /// - stride: The stride of the model. + /// - kernel: The kernel size of the model. + /// - Returns: The built model. + /// + static func build(size: Int, stride: Int, kernel: Int) -> Model + { + let context = ModelContext(name: "ModelTestDeConvSK", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: size, + height: size, + params: params + ) + + layer = Deconvolution2D( + layerPrev: layer, + size: kernel, nbChannels: 5, stride: stride, + activation: nil, biases: true, bn: false, + params: params + ) + + var head: Layer1D = AvgPool2D( + layerPrev: layer, params: params + ) + + head = try! FullyConnected( + layerPrev: head, + nbNeurons: 1, + activation: nil, + biases: true, + params: params + ) + + let model = Model(model: context.model, modelsPrev: []) + + // Load weights from `PyTorch`. + let pythonLib = Python.import("python_lib") + let data = pythonLib.load_deconv_sk_weights(stride, kernel) + + let weights = [[Float]](data.tuple2.0)! + super.initWeights(model: model, weights: weights) + + return model + } +} + +/// Model to test against PyTorch. +class ModelTestCat +{ + /// + /// Create the model and import weights from PyTorch. + /// + /// Principle features: + /// - Concat2D + /// + /// - Parameter size: The size of the input data. + /// - Returns: The built model. + /// + static func build(_ size: Int) -> Model + { + let context = ModelContext(name: "ModelTestCat", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: size, + height: size, + params: params + ) + + let layer1: Layer2D = Convolution2D( + layerPrev: layer, + size: 1, nbChannels: 6, stride: 1, + activation: nil, biases: true, bn: false, + params: params + ) + let layer2: Layer2D = Convolution2D( + layerPrev: layer, + size: 1, nbChannels: 9, stride: 1, + activation: nil, biases: true, bn: false, + params: params + ) + + layer = try! Concat2D( + layersPrev: [layer1, layer2], params: params + ) + + var head: Layer1D = AvgPool2D( + layerPrev: layer, params: params + ) + + head = try! FullyConnected( + layerPrev: head, + nbNeurons: 1, + activation: nil, + biases: true, + params: params + ) + + let model = Model(model: context.model, modelsPrev: []) + + // Load weights from `PyTorch`. + let pythonLib = Python.import("python_lib") + let data = pythonLib.load_cat_weights() + + let weights = [[Float]](data.tuple2.0)! + + // Apply weights on the `GrAIdient` model's layers. + var cur = 0 + for num_layer in 0.. Model + { + let context = ModelContext(name: "ModelTestResize", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: sizeInput, + height: sizeInput, + params: params + ) + + layer = Convolution2D( + layerPrev: layer, + size: 1, nbChannels: 5, stride: 1, + activation: nil, biases: true, bn: false, + params: params + ) + + layer = try! ResizeBilinear( + layerPrev: layer, + dimension: sizeOutput, + params: params + ) + + var head: Layer1D = AvgPool2D( + layerPrev: layer, params: params + ) + + head = try! FullyConnected( + layerPrev: head, + nbNeurons: 1, + activation: nil, + biases: true, + params: params + ) + + let model = Model(model: context.model, modelsPrev: []) + + // Load weights from `PyTorch`. + let pythonLib = Python.import("python_lib") + let data = pythonLib.load_resize_weights(sizeOutput) + + let weights = [[Float]](data.tuple2.0)! + super.initWeights(model: model, weights: weights) + + return model + } +} + +/// Model to test against PyTorch. +class ModelTestResizeBilinearPad: ModelTestResize +{ + /// + /// Create the model and import weights from PyTorch. + /// + /// Principle features: + /// - ResizeBilinearPad + /// + /// - Parameters: + /// - sizeInput: The size of the input data. + /// - sizeOutput: The output size of the resize operation. + /// - Returns: The built model. + /// + static func build(sizeInput: Int, sizeOutput: Int) -> Model + { + let context = ModelContext(name: "ModelTestResize", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: sizeInput, + height: sizeInput, + params: params + ) + + layer = Convolution2D( + layerPrev: layer, + size: 1, nbChannels: 5, stride: 1, + activation: nil, biases: true, bn: false, + params: params + ) + + layer = try! ResizeBilinearPad( + layerPrev: layer, + scalesList: [Double(sizeOutput) / Double(sizeInput)], + padValue: 0.0, + params: params + ) + + var head: Layer1D = AvgPool2D( + layerPrev: layer, params: params + ) + + head = try! FullyConnected( + layerPrev: head, + nbNeurons: 1, + activation: nil, + biases: true, + params: params + ) + + let model = Model(model: context.model, modelsPrev: []) + + // Load weights from `PyTorch`. + let pythonLib = Python.import("python_lib") + let data = pythonLib.load_resize_weights(sizeOutput) + + let weights = [[Float]](data.tuple2.0)! + super.initWeights(model: model, weights: weights) + + return model + } +} + /// Model to test against PyTorch. -class ModelTest4 +class ModelTestResizeBilinearCrop: ModelTestResize { /// /// Create the model and import weights from PyTorch. /// /// Principle features: - /// - Deconvolution with odd kernel and stride + /// - ResizeBilinearCrop /// - /// - Parameter size: The size of the input data. + /// - Parameters: + /// - sizeInput: The size of the input data. + /// - sizeOutput: The output size of the resize operation. /// - Returns: The built model. /// - static func build(_ size: Int) -> Model + static func build(sizeInput: Int, sizeOutput: Int) -> Model { - let context = ModelContext(name: "ModelTest4", curID: 0) + let context = ModelContext(name: "ModelTestResize", curID: 0) let params = GrAI.Model.Params(context: context) var layer: Layer2D layer = Input2D( nbChannels: 3, - width: size, - height: size, + width: sizeInput, + height: sizeInput, params: params ) - layer = Deconvolution2D( + layer = Convolution2D( layerPrev: layer, - size: 3, nbChannels: 5, stride: 2, + size: 1, nbChannels: 5, stride: 1, activation: nil, biases: true, bn: false, params: params ) + layer = try! ResizeBilinearCrop( + layerPrev: layer, + scalesList: [Double(sizeOutput) / Double(sizeInput)], + params: params + ) + var head: Layer1D = AvgPool2D( layerPrev: layer, params: params ) - head = FullyConnected( + head = try! FullyConnected( layerPrev: head, nbNeurons: 1, activation: nil, @@ -352,7 +783,64 @@ class ModelTest4 // Load weights from `PyTorch`. let pythonLib = Python.import("python_lib") - let data = pythonLib.load_test4_weights() + let data = pythonLib.load_resize_weights(sizeOutput) + + let weights = [[Float]](data.tuple2.0)! + super.initWeights(model: model, weights: weights) + + return model + } +} + +/// Model to test against PyTorch. +class ModelTestPatchConv +{ + /// + /// Create the model and import weights from PyTorch. + /// + /// Principle features: + /// - FullyConnectedPatch + /// - AvgPoolSeq + /// + /// - Parameters: + /// - size: The size of the input data. + /// - patch: The kernel split size of the input data. + /// - Returns: The built model. + /// + static func build(size: Int, patch: Int) -> Model + { + let context = ModelContext(name: "ModelTestPatchConv", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: size, + height: size, + params: params + ) + + let layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: patch, nbNeurons: 5, + activation: nil, biases: true, + params: params + ) + + var head: Layer1D = AvgPoolSeq( + layerPrev: layerSeq, params: params + ) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: nil, biases: true, + params: params + ) + + let model = Model(model: context.model, modelsPrev: []) + + // Load weights from `PyTorch`. + let pythonLib = Python.import("python_lib") + let data = pythonLib.load_patch_conv_weights(size, patch) let weights = [[Float]](data.tuple2.0)! @@ -361,16 +849,15 @@ class ModelTest4 for num_layer in 0.. Model + static func build(size: Int, patch: Int) -> Model { - let context = ModelContext(name: "ModelTest5", curID: 0) + let context = ModelContext(name: "ModelTestAttention1", curID: 0) let params = GrAI.Model.Params(context: context) var layer: Layer2D @@ -411,22 +903,51 @@ class ModelTest5 params: params ) - layer = Deconvolution2D( - layerPrev: layer, - size: 2, nbChannels: 5, stride: 2, - activation: nil, biases: true, bn: false, + var layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: patch, nbNeurons: 5, + activation: nil, biases: true, params: params ) - var head: Layer1D = AvgPool2D( - layerPrev: layer, params: params + let query = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 5, + activation: nil, biases: true, + params: params + ) + let key = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 5, + activation: nil, biases: true, + params: params + ) + let value = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 5, + activation: nil, biases: true, + params: params ) - head = FullyConnected( - layerPrev: head, - nbNeurons: 1, - activation: nil, - biases: true, + var score: LayerSeq = try! QuerySeq( + query: query, key: key, nbHeads: 1, params: params + ) + score = try! SoftmaxSeq(layerPrev: score, nbHeads: 1, params: params) + + layerSeq = try! ValueSeq( + value: value, score: score, nbHeads: 1, + params: params + ) + + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 5, + activation: nil, biases: true, + params: params + ) + + var head: Layer1D = AvgPoolSeq( + layerPrev: layerSeq, params: params + ) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: nil, biases: true, params: params ) @@ -434,7 +955,7 @@ class ModelTest5 // Load weights from `PyTorch`. let pythonLib = Python.import("python_lib") - let data = pythonLib.load_test5_weights() + let data = pythonLib.load_attention1_weights(size, patch) let weights = [[Float]](data.tuple2.0)! @@ -443,16 +964,24 @@ class ModelTest5 for num_layer in 0.. Model + static func build(size: Int, patch: Int) -> Model { - let context = ModelContext(name: "ModelTest6", curID: 0) + let context = ModelContext(name: "ModelTestAttention2", curID: 0) let params = GrAI.Model.Params(context: context) var layer: Layer2D @@ -493,22 +1027,56 @@ class ModelTest6 params: params ) - layer = Deconvolution2D( - layerPrev: layer, - size: 3, nbChannels: 5, stride: 1, - activation: nil, biases: true, bn: false, + var layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: patch, nbNeurons: 6, + activation: nil, biases: true, params: params ) - var head: Layer1D = AvgPool2D( - layerPrev: layer, params: params + let query: LayerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 6, + activation: nil, biases: true, + params: params + ) + let key: LayerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 6, + activation: nil, biases: true, + params: params + ) + let value: LayerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 6, + activation: nil, biases: true, + params: params ) - head = FullyConnected( - layerPrev: head, - nbNeurons: 1, - activation: nil, - biases: true, + let nbHeads = 3 + layerSeq = try! QuerySeq( + query: query, key: key, nbHeads: nbHeads, + params: params + ) + layerSeq = try! SoftmaxSeq( + layerPrev: layerSeq, nbHeads: nbHeads, + params: params + ) + + layerSeq = try! ValueSeq( + value: value, score: layerSeq, nbHeads: nbHeads, + params: params + ) + + layerSeq = FullyConnectedSeq( + layerPrev: layerSeq, nbNeurons: 6, + activation: nil, biases: true, + params: params + ) + + var head: Layer1D = AvgPoolSeq( + layerPrev: layerSeq, params: params + ) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: nil, biases: true, params: params ) @@ -516,7 +1084,7 @@ class ModelTest6 // Load weights from `PyTorch`. let pythonLib = Python.import("python_lib") - let data = pythonLib.load_test6_weights() + let data = pythonLib.load_attention2_weights(size, patch) let weights = [[Float]](data.tuple2.0)! @@ -525,16 +1093,24 @@ class ModelTest6 for num_layer in 0.. Model + static func build(size: Int, patch: Int) -> Model { - let context = ModelContext(name: "ModelTest7", curID: 0) + let context = ModelContext(name: "ModelTestLayerNorm", curID: 0) let params = GrAI.Model.Params(context: context) var layer: Layer2D @@ -575,22 +1153,23 @@ class ModelTest7 params: params ) - layer = Deconvolution2D( - layerPrev: layer, - size: 2, nbChannels: 5, stride: 1, - activation: nil, biases: true, bn: false, + var layerSeq: LayerSeq = try! FullyConnectedPatch( + layerPrev: layer, patch: patch, nbNeurons: 5, + activation: nil, biases: true, params: params ) - var head: Layer1D = AvgPool2D( - layerPrev: layer, params: params + layerSeq = LayerNormSeq( + layerPrev: layerSeq, activation: nil, params: params ) - head = FullyConnected( - layerPrev: head, - nbNeurons: 1, - activation: nil, - biases: true, + var head: Layer1D = AvgPoolSeq( + layerPrev: layerSeq, params: params + ) + + head = try! FullyConnected( + layerPrev: head, nbNeurons: 1, + activation: nil, biases: true, params: params ) @@ -598,10 +1177,60 @@ class ModelTest7 // Load weights from `PyTorch`. let pythonLib = Python.import("python_lib") - let data = pythonLib.load_test7_weights() + let data = pythonLib.load_layer_norm_weights(size, patch) let weights = [[Float]](data.tuple2.0)! + // Apply weights on the `GrAIdient` model's layers. + var cur = 0 + for num_layer in 0.. Model { - let context = ModelContext(name: "ModelTest8", curID: 0) + let context = ModelContext(name: "ModelTestAutoEncoder1", curID: 0) let params = GrAI.Model.Params(context: context) var layer: Layer2D @@ -657,28 +1285,73 @@ class ModelTest8 params: params ) - let layer1: Layer2D = Convolution2D( + layer = Convolution2D( layerPrev: layer, - size: 1, nbChannels: 6, stride: 1, - activation: nil, biases: true, bn: false, + size: 3, nbChannels: 5, stride: 2, + activation: nil, + biases: true, bn: false, params: params ) - let layer2: Layer2D = Convolution2D( + + layer = Deconvolution2D( layerPrev: layer, - size: 1, nbChannels: 9, stride: 1, + size: 2, nbChannels: 3, stride: 2, activation: nil, biases: true, bn: false, params: params ) - layer = Concat2D( - layersPrev: [layer1, layer2], params: params + let model = Model(model: context.model, modelsPrev: []) + + // Load weights from `PyTorch`. + let pythonLib = Python.import("python_lib") + let data = pythonLib.load_auto_encoder1_weights() + + let weights = [[Float]](data.tuple2.0)! + super.initWeights(model: model, weights: weights) + + return model + } +} + +/// Model to test against PyTorch. +class ModelTestGram +{ + /// + /// Create the model and import weights from PyTorch. + /// + /// Principle features: + /// - SelfCorrelate + /// - Normalize12 + /// - SimilarityBatchError + /// + /// - Parameter size: The size of the input data. + /// - Returns: The built model. + /// + static func build(_ size: Int) -> Model + { + let context = ModelContext(name: "ModelTestGram", curID: 0) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D + layer = Input2D( + nbChannels: 3, + width: size, + height: size, + params: params ) - var head: Layer1D = AvgPool2D( + layer = Convolution2D( + layerPrev: layer, + size: 1, nbChannels: 5, stride: 1, + activation: nil, biases: true, bn: false, + params: params + ) + + let head: Layer1D = AvgPool2D( layerPrev: layer, params: params ) - head = FullyConnected( + _ = try! FullyConnected( layerPrev: head, nbNeurons: 1, activation: nil, @@ -690,7 +1363,7 @@ class ModelTest8 // Load weights from `PyTorch`. let pythonLib = Python.import("python_lib") - let data = pythonLib.load_test8_weights() + let data = pythonLib.load_gram_weights() let weights = [[Float]](data.tuple2.0)! diff --git a/Tests/GrAITorchTests/Base/Utils.swift b/Tests/GrAITorchTests/Base/Utils.swift index 3d0f14d0..439b0e13 100644 --- a/Tests/GrAITorchTests/Base/Utils.swift +++ b/Tests/GrAITorchTests/Base/Utils.swift @@ -59,6 +59,21 @@ func getInputData(_ size: Int) -> [Float] return [Float](data)! } +/// +/// Get data image from Python. +/// +/// - Parameters: +/// - size: The size of the image. +/// - batchSize: The number of elements in the batch. +/// - Returns: The data image. +/// +func getBatchData(size: Int, batchSize: Int) -> [Float] +{ + let pythonLib = Python.import("python_lib") + let data = pythonLib.get_batch_data(size, batchSize) + return [Float](data)! +} + /// /// Get data "complex" image from Python. /// @@ -73,106 +88,202 @@ func getComplexData(_ size: Int) -> [Float] } /// -/// Get gradient norm computed with PyTorch for ModelTest1. +/// Get gradient norm computed with PyTorch for ModelTestConv1. /// /// - Parameter size: The size of the input data. /// - Returns: The gradient norm. /// -func computeTest1GradNorm(_ size: Int) -> Float +func computeConv1GradNorm(_ size: Int) -> Float { let pythonLib = Python.import("python_lib") - let data = pythonLib.compute_test1_grad_norm(size) + let data = pythonLib.compute_conv1_grad_norm(size) return Float(data)! } /// -/// Get gradient norm computed with PyTorch for ModelTest2. +/// Get gradient norm computed with PyTorch for ModelTestConv2. /// /// - Parameter size: The size of the input data. /// - Returns: The gradient norm. /// -func computeTest2GradNorm(_ size: Int) -> Float +func computeConv2GradNorm(_ size: Int) -> Float { let pythonLib = Python.import("python_lib") - let data = pythonLib.compute_test2_grad_norm(size) + let data = pythonLib.compute_conv2_grad_norm(size) return Float(data)! } /// -/// Get gradient norm computed with PyTorch for ModelTest3. +/// Get gradient norm computed with PyTorch for ModelTestFFT. /// /// - Parameter size: The size of the input data. /// - Returns: The gradient norm. /// -func computeTest3GradNorm(_ size: Int) -> Float +func computeFFTGradNorm(_ size: Int) -> Float { let pythonLib = Python.import("python_lib") - let data = pythonLib.compute_test3_grad_norm(size) + let data = pythonLib.compute_fft_grad_norm(size) return Float(data)! } /// -/// Get gradient norm computed with PyTorch for ModelTest4. +/// Get gradient norm computed with PyTorch for ModelTestConvSK. /// -/// - Parameter size: The size of the input data. +/// - Parameters: +/// - size: The size of the input data. +/// - stride: The stride of the model. +/// - kernel: The kernel size of the model. /// - Returns: The gradient norm. /// -func computeTest4GradNorm(_ size: Int) -> Float +func computeConvSKGradNorm(size: Int, stride: Int, kernel: Int) -> Float { let pythonLib = Python.import("python_lib") - let data = pythonLib.compute_test4_grad_norm(size) + let data = pythonLib.compute_conv_sk_grad_norm(size, stride, kernel) return Float(data)! } /// -/// Get gradient norm computed with PyTorch for ModelTest5. +/// Get gradient norm computed with PyTorch for ModelTestDeConvSK. /// -/// - Parameter size: The size of the input data. +//// - Parameters: +/// - size: The size of the input data. +/// - stride: The stride of the model. +/// - kernel: The kernel size of the model. /// - Returns: The gradient norm. /// -func computeTest5GradNorm(_ size: Int) -> Float +func computeDeConvSKGradNorm(size: Int, stride: Int, kernel: Int) -> Float { let pythonLib = Python.import("python_lib") - let data = pythonLib.compute_test5_grad_norm(size) + let data = pythonLib.compute_deconv_sk_grad_norm(size, stride, kernel) return Float(data)! } /// -/// Get gradient norm computed with PyTorch for ModelTest6. +/// Get gradient norm computed with PyTorch for ModelTestCat. /// /// - Parameter size: The size of the input data. /// - Returns: The gradient norm. /// -func computeTest6GradNorm(_ size: Int) -> Float +func computeCatGradNorm(_ size: Int) -> Float +{ + let pythonLib = Python.import("python_lib") + let data = pythonLib.compute_cat_grad_norm(size) + return Float(data)! +} + +/// +/// Get gradient norm computed with PyTorch for ModelTestResize. +/// +/// - Parameters: +/// - sizeInput: The size of the input data. +/// - sizeOutput: The output size of the resize operation. +/// - Returns: The gradient norm. +/// +func computeResizeGradNorm(sizeInput: Int, sizeOutput: Int) -> Float +{ + let pythonLib = Python.import("python_lib") + let data = pythonLib.compute_resize_grad_norm(sizeInput, sizeOutput) + return Float(data)! +} + +/// +/// Get gradient norm computed with PyTorch for ModelTestPatchConv. +/// +/// - Parameters: +/// - size: The size of the input data. +/// - patch: The kernel split size of the input data. +/// - Returns: The gradient norm. +/// +func computePatchConvGradNorm(size: Int, patch: Int) -> Float +{ + let pythonLib = Python.import("python_lib") + let data = pythonLib.compute_patch_conv_grad_norm(size, patch) + return Float(data)! +} + +/// +/// Get gradient norm computed with PyTorch for ModelTestAttention1. +/// +/// - Parameters: +/// - size: The size of the input data. +/// - patch: The kernel split size of the input data. +/// - Returns: The gradient norm. +/// +func computeAttention1GradNorm(size: Int, patch: Int) -> Float { let pythonLib = Python.import("python_lib") - let data = pythonLib.compute_test6_grad_norm(size) + let data = pythonLib.compute_attention1_grad_norm(size, patch) return Float(data)! } /// -/// Get gradient norm computed with PyTorch for ModelTest7. +/// Get gradient norm computed with PyTorch for ModelTestAttention2. +/// +/// - Parameters: +/// - size: The size of the input data. +/// - patch: The kernel split size of the input data. +/// - Returns: The gradient norm. +/// +func computeAttention2GradNorm(size: Int, patch: Int) -> Float +{ + let pythonLib = Python.import("python_lib") + let data = pythonLib.compute_attention2_grad_norm(size, patch) + return Float(data)! +} + +/// +/// Get gradient norm computed with PyTorch for ModelTestLayerNorm. +/// +/// - Parameters: +/// - size: The size of the input data. +/// - patch: The kernel split size of the input data. +/// - Returns: The gradient norm. +/// +func computeLayerNormGradNorm(size: Int, patch: Int) -> Float +{ + let pythonLib = Python.import("python_lib") + let data = pythonLib.compute_layer_norm_grad_norm(size, patch) + return Float(data)! +} + +/// +/// Get gradient norm computed with PyTorch for ModelTestAutoEncoder1. /// /// - Parameter size: The size of the input data. /// - Returns: The gradient norm. /// -func computeTest7GradNorm(_ size: Int) -> Float +func computeAutoEncoder1GradNorm(_ size: Int) -> Float { let pythonLib = Python.import("python_lib") - let data = pythonLib.compute_test7_grad_norm(size) + let data = pythonLib.compute_auto_encoder1_grad_norm(size) return Float(data)! } /// -/// Get gradient norm computed with PyTorch for ModelTest8. +/// Get gradient norm computed with PyTorch for ModelTestAutoEncoder2. /// /// - Parameter size: The size of the input data. /// - Returns: The gradient norm. /// -func computeTest8GradNorm(_ size: Int) -> Float +func computeAutoEncoder2GradNorm(_ size: Int) -> Float +{ + let pythonLib = Python.import("python_lib") + let data = pythonLib.compute_auto_encoder2_grad_norm(size) + return Float(data)! +} + +/// +/// Get gradient norm computed with PyTorch for ModelTestGram. +/// +/// - Parameters: +/// - size: The size of the input data. +/// - batchSize: The number of images in the batch. +/// - Returns: The gradient norm. +/// +func computeGramGradNorm(size: Int, batchSize: Int) -> Float { let pythonLib = Python.import("python_lib") - let data = pythonLib.compute_test8_grad_norm(size) + let data = pythonLib.compute_gram_grad_norm(size, batchSize) return Float(data)! } diff --git a/Tests/GrAITorchTests/Base/python_lib/__init__.py b/Tests/GrAITorchTests/Base/python_lib/__init__.py index f7db7feb..bb7395ee 100644 --- a/Tests/GrAITorchTests/Base/python_lib/__init__.py +++ b/Tests/GrAITorchTests/Base/python_lib/__init__.py @@ -1,41 +1,62 @@ from python_lib.gradient import ( get_input_data, + get_batch_data, get_complex_data, - compute_test1_grad_norm, - compute_test2_grad_norm, - compute_test3_grad_norm, - compute_test4_grad_norm, - compute_test5_grad_norm, - compute_test6_grad_norm, - compute_test7_grad_norm, - compute_test8_grad_norm, + compute_conv1_grad_norm, + compute_conv2_grad_norm, + compute_fft_grad_norm, + compute_conv_sk_grad_norm, + compute_deconv_sk_grad_norm, + compute_cat_grad_norm, + compute_resize_grad_norm, + compute_patch_conv_grad_norm, + compute_attention1_grad_norm, + compute_attention2_grad_norm, + compute_layer_norm_grad_norm, + compute_auto_encoder1_grad_norm, + compute_gram_grad_norm, ) from python_lib.weight import ( - load_test1_weights, - load_test2_weights, - load_test4_weights, - load_test5_weights, - load_test6_weights, - load_test7_weights, - load_test8_weights, + load_conv1_weights, + load_conv2_weights, + load_conv_sk_weights, + load_deconv_sk_weights, + load_cat_weights, + load_resize_weights, + load_patch_conv_weights, + load_attention1_weights, + load_attention2_weights, + load_layer_norm_weights, + load_auto_encoder1_weights, + load_gram_weights, ) __all__ = [ "get_input_data", + "get_batch_data", "get_complex_data", - "compute_test1_grad_norm", - "compute_test2_grad_norm", - "compute_test3_grad_norm", - "compute_test4_grad_norm", - "compute_test5_grad_norm", - "compute_test6_grad_norm", - "compute_test7_grad_norm", - "compute_test8_grad_norm", - "load_test1_weights", - "load_test2_weights", - "load_test4_weights", - "load_test5_weights", - "load_test6_weights", - "load_test7_weights", - "load_test8_weights", + "compute_conv1_grad_norm", + "compute_conv2_grad_norm", + "compute_fft_grad_norm", + "compute_conv_sk_grad_norm", + "compute_deconv_sk_grad_norm", + "compute_cat_grad_norm", + "compute_resize_grad_norm", + "compute_patch_conv_grad_norm", + "compute_attention1_grad_norm", + "compute_attention2_grad_norm", + "compute_layer_norm_grad_norm", + "compute_auto_encoder1_grad_norm", + "compute_gram_grad_norm", + "load_conv1_weights", + "load_conv2_weights", + "load_conv_sk_weights", + "load_deconv_sk_weights", + "load_cat_weights", + "load_patch_conv_weights", + "load_attention1_weights", + "load_attention2_weights", + "load_layer_norm_weights", + "load_auto_encoder1_weights", + "load_gram_weights", ] diff --git a/Tests/GrAITorchTests/Base/python_lib/gradient.py b/Tests/GrAITorchTests/Base/python_lib/gradient.py index b3bc7eb1..d8cc1ea3 100644 --- a/Tests/GrAITorchTests/Base/python_lib/gradient.py +++ b/Tests/GrAITorchTests/Base/python_lib/gradient.py @@ -5,14 +5,19 @@ from torchvision.transforms import ToTensor from python_lib.model import ( - ModelTest1, - ModelTest2, - ModelTest3, - ModelTest4, - ModelTest5, - ModelTest6, - ModelTest7, - ModelTest8, + ModelTestConv1, + ModelTestConv2, + ModelTestFFT, + ModelTestConvSK, + ModelTestDeConvSK, + ModelTestCat, + ModelTestResize, + ModelTestPatchConv, + ModelTestAttention1, + ModelTestAttention2, + ModelTestLayerNorm, + ModelTestAutoEncoder1, + ModelTestGram, ) @@ -102,6 +107,61 @@ def get_input_data(size: int) -> List[float]: return data +def _build_batch_data(size: int, nb_batch: int) -> np.ndarray: + """ + Build data image. + + Parameters + ---------- + size: int + The size of the image to build. + nb_batch: int + Number of images in the batch. + + Returns + ------- + _: np.ndarray + The batch images with 3 channels. + """ + img_array = np.zeros((nb_batch, size, size, 3)) + for batch in range(nb_batch): + for depth in range(3): + for row in range(size): + if depth == 0: + img_array[batch, row, :, depth] = \ + (np.arange(0, size, 1) + row) / (2 * size) \ + + batch + elif depth == 1: + img_array[batch, row, :, depth] = \ + (np.arange(size - 1, -1, -1) + row) / (2 * size) \ + + batch + else: + img_array[batch, row, :, depth] = \ + (np.arange(0, size, 1) + size - 1 - row) / (2 * size) \ + + batch + return img_array + + +def get_batch_data(size: int, nb_batch: int) -> List[float]: + """ + Get data image and flatten it. + + Parameters + ---------- + size: int + The size of the image to build. + nb_batch: int + The number of images in the batch. + + Returns + ------- + _: List[float] + The images with 3 channels flattened. + """ + data: List[float] = _build_batch_data(size, nb_batch).flatten().tolist() + return data + + def _build_complex_data(size: int) -> np.ndarray: """ Build data "complex" image. @@ -166,6 +226,8 @@ def _compute_grad_norm(input: torch.Tensor, model: torch.nn.Module) -> float: Parameters ---------- + input: torch.Tensor + The input tensor. model: torch.nn.Module The model to test. @@ -190,9 +252,9 @@ def _compute_grad_norm(input: torch.Tensor, model: torch.nn.Module) -> float: return gradient_norm -def compute_test1_grad_norm(size: int) -> float: +def compute_conv1_grad_norm(size: int) -> float: """ - Compute the gradient norm of one backward pass of ModelTest1. + Compute the gradient norm of one backward pass of ModelTestConv1. Parameters ---------- @@ -207,13 +269,13 @@ def compute_test1_grad_norm(size: int) -> float: torch.manual_seed(42) img_array = _build_input_data(size) img_tensor = ToTensor()(img_array).type(torch.float32) - model = ModelTest1().eval().cpu() + model = ModelTestConv1().eval().cpu() return _compute_grad_norm(img_tensor, model) -def compute_test2_grad_norm(size: int) -> float: +def compute_conv2_grad_norm(size: int) -> float: """ - Compute the gradient norm of one backward pass of ModelTest2. + Compute the gradient norm of one backward pass of ModelTestConv2. Parameters ---------- @@ -228,13 +290,13 @@ def compute_test2_grad_norm(size: int) -> float: torch.manual_seed(42) img_array = _build_input_data(size) img_tensor = ToTensor()(img_array).type(torch.float32) - model = ModelTest2().eval().cpu() + model = ModelTestConv2().eval().cpu() return _compute_grad_norm(img_tensor, model) -def compute_test3_grad_norm(size: int) -> float: +def compute_fft_grad_norm(size: int) -> float: """ - Compute the gradient norm of one backward pass of ModelTest3. + Compute the gradient norm of one backward pass of ModelTestFFT. Parameters ---------- @@ -249,18 +311,25 @@ def compute_test3_grad_norm(size: int) -> float: torch.manual_seed(42) img_array = _build_complex_data(size) img_tensor = ToTensor()(img_array).type(torch.float32) - model = ModelTest3(size).eval().cpu() + model = ModelTestFFT(size).eval().cpu() return _compute_grad_norm(img_tensor, model) -def compute_test4_grad_norm(size: int) -> float: +def compute_conv_sk_grad_norm( + size: int, stride: int, kernel: int +) -> float: """ - Compute the gradient norm of one backward pass of ModelTest4. + Compute the gradient norm of one backward pass of + ModelTestConvSK. Parameters ---------- size: int The size of the input data. + stride: int + The stride of the model. + kernel: int + The kernel size of the model. Returns ------- @@ -270,18 +339,27 @@ def compute_test4_grad_norm(size: int) -> float: torch.manual_seed(42) img_array = _build_input_data(size) img_tensor = ToTensor()(img_array).type(torch.float32) - model = ModelTest4().eval().cpu() + model = ModelTestConvSK( + stride=stride, kernel=kernel + ).eval().cpu() return _compute_grad_norm(img_tensor, model) -def compute_test5_grad_norm(size: int) -> float: +def compute_deconv_sk_grad_norm( + size: int, stride: int, kernel: int +) -> float: """ - Compute the gradient norm of one backward pass of ModelTest5. + Compute the gradient norm of one backward pass of + ModelTestDeConvSK. Parameters ---------- size: int The size of the input data. + stride: int + The stride of the model. + kernel: int + The kernel size of the model. Returns ------- @@ -291,13 +369,15 @@ def compute_test5_grad_norm(size: int) -> float: torch.manual_seed(42) img_array = _build_input_data(size) img_tensor = ToTensor()(img_array).type(torch.float32) - model = ModelTest5().eval().cpu() + model = ModelTestDeConvSK( + stride=stride, kernel=kernel + ).eval().cpu() return _compute_grad_norm(img_tensor, model) -def compute_test6_grad_norm(size: int) -> float: +def compute_cat_grad_norm(size: int) -> float: """ - Compute the gradient norm of one backward pass of ModelTest6. + Compute the gradient norm of one backward pass of ModelTestCat. Parameters ---------- @@ -312,18 +392,43 @@ def compute_test6_grad_norm(size: int) -> float: torch.manual_seed(42) img_array = _build_input_data(size) img_tensor = ToTensor()(img_array).type(torch.float32) - model = ModelTest6().eval().cpu() + model = ModelTestCat().eval().cpu() return _compute_grad_norm(img_tensor, model) -def compute_test7_grad_norm(size: int) -> float: +def compute_resize_grad_norm(size_input: int, size_output) -> float: """ - Compute the gradient norm of one backward pass of ModelTest7. + Compute the gradient norm of one backward pass of ModelTestResize. + + Parameters + ---------- + size_input: int + The size of the input data. + size_output: int + The size of the output resize operation. + + Returns + ------- + _: float + The gradient norm. + """ + torch.manual_seed(42) + img_array = _build_input_data(size_input) + img_tensor = ToTensor()(img_array).type(torch.float32) + model = ModelTestResize(size_output).eval().cpu() + return _compute_grad_norm(img_tensor, model) + + +def compute_patch_conv_grad_norm(size: int, patch: int) -> float: + """ + Compute the gradient norm of one backward pass of ModelTestPathConv. Parameters ---------- size: int The size of the input data. + patch: int + kernel split size of the input data. Returns ------- @@ -333,18 +438,20 @@ def compute_test7_grad_norm(size: int) -> float: torch.manual_seed(42) img_array = _build_input_data(size) img_tensor = ToTensor()(img_array).type(torch.float32) - model = ModelTest7().eval().cpu() + model = ModelTestPatchConv(size=size, patch=patch).eval().cpu() return _compute_grad_norm(img_tensor, model) -def compute_test8_grad_norm(size: int) -> float: +def compute_attention1_grad_norm(size: int, patch: int) -> float: """ - Compute the gradient norm of one backward pass of ModelTest8. + Compute the gradient norm of one backward pass of ModelTestAttention1. Parameters ---------- size: int The size of the input data. + patch: int + kernel split size of the input data. Returns ------- @@ -354,5 +461,175 @@ def compute_test8_grad_norm(size: int) -> float: torch.manual_seed(42) img_array = _build_input_data(size) img_tensor = ToTensor()(img_array).type(torch.float32) - model = ModelTest8().eval().cpu() + model = ModelTestAttention1(size=size, patch=patch).eval().cpu() return _compute_grad_norm(img_tensor, model) + + +def compute_attention2_grad_norm(size: int, patch: int) -> float: + """ + Compute the gradient norm of one backward pass of ModelTestAttention2. + + Parameters + ---------- + size: int + The size of the input data. + patch: int + kernel split size of the input data. + + Returns + ------- + _: float + The gradient norm. + """ + torch.manual_seed(42) + img_array = _build_input_data(size) + img_tensor = ToTensor()(img_array).type(torch.float32) + model = ModelTestAttention2(size=size, patch=patch).eval().cpu() + return _compute_grad_norm(img_tensor, model) + + +def compute_layer_norm_grad_norm(size: int, patch: int) -> float: + """ + Compute the gradient norm of one backward pass of ModelTestLayerNorm. + + Parameters + ---------- + size: int + The size of the input data. + patch: int + kernel split size of the input data. + + Returns + ------- + _: float + The gradient norm. + """ + torch.manual_seed(42) + img_array = _build_input_data(size) + img_tensor = ToTensor()(img_array).type(torch.float32) + model = ModelTestLayerNorm(size=size, patch=patch).eval().cpu() + return _compute_grad_norm(img_tensor, model) + + +def _compute_auto_encoder_grad_norm( + input: torch.Tensor, model: torch.nn.Module +) -> float: + """ + Compute the gradient norm of one backward pass in a specific context. + + Parameters + ---------- + model: torch.nn.Module + The model to test. + + Returns + ------- + _: float + The gradient norm. + """ + img_var = Variable(input, requires_grad=True) + gradient = GetGradient(img_var) + + x = img_var + x = x[None, :] + gt = x.detach() + x = model(x) + + loss = torch.nn.MSELoss()(x, gt) + loss.backward() + + gradient_norm = gradient.gradient_norm + gradient.close() + return gradient_norm + + +def compute_auto_encoder1_grad_norm(size: int) -> float: + """ + Compute the gradient norm of one backward pass of ModelTestAutoEncoder1. + + Parameters + ---------- + size: int + The size of the input data. + + Returns + ------- + _: float + The gradient norm. + """ + torch.manual_seed(42) + img_array = _build_input_data(size) + img_tensor = ToTensor()(img_array).type(torch.float32) + model = ModelTestAutoEncoder1().eval().cpu() + return _compute_auto_encoder_grad_norm(img_tensor, model) + + +def _compute_gram_grad_norm( + input: torch.Tensor, + model: torch.nn.Module +) -> float: + """ + Compute the gradient norm of one backward pass in a specific context. + + Parameters + ---------- + input: torch.Tensor + The input tensor. + model: torch.nn.Module + The model to test. + + Returns + ------- + _: float + The gradient norm. + """ + img_var = Variable(input, requires_grad=True) + gradient = GetGradient(img_var) + + x = img_var + x, features = model(x) + + nb_batch, nb_channels, _, _ = features.shape + similarity = features.view(nb_batch, nb_channels, -1) + similarity = torch.matmul(similarity, torch.transpose(similarity, 1, 2)) + similarity = torch.nn.functional.normalize(similarity, p=2, dim=(1, 2)) + similarity = sum([ sum([(similarity[i] * similarity[j]).sum() + for j in range(nb_batch) if j != i]) + for i in range(nb_batch)] + ) / nb_batch + + x = torch.nn.MSELoss()(x, torch.zeros_like(x)) + loss = 1.0 / 2.0 * x + similarity + loss.backward() + + gradient_norm = gradient.gradient_norm + gradient.close() + return gradient_norm + + +def compute_gram_grad_norm(size: int, nb_batch: int) -> float: + """ + Compute the gradient norm of one backward pass of ModelTestGram. + + Parameters + ---------- + size: int + The size of the input data. + nb_batch: int + The number of images in the batch. + + Returns + ------- + _: float + The gradient norm. + """ + torch.manual_seed(42) + img_array = _build_batch_data(size, nb_batch) + + images = [] + for batch in range(nb_batch): + images.append(ToTensor()(img_array[batch]).type(torch.float32)[None,:]) + img_tensor = torch.cat(images) + + model = ModelTestGram().eval().cpu() + return _compute_gram_grad_norm(img_tensor, model) diff --git a/Tests/GrAITorchTests/Base/python_lib/model.py b/Tests/GrAITorchTests/Base/python_lib/model.py index 6d5e76af..8c02679f 100644 --- a/Tests/GrAITorchTests/Base/python_lib/model.py +++ b/Tests/GrAITorchTests/Base/python_lib/model.py @@ -1,8 +1,10 @@ +import math import torch import numpy as np +from typing import Tuple -class ModelTest1(torch.nn.Module): +class ModelTestConv1(torch.nn.Module): """ Model to test. Principle features: @@ -78,7 +80,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return x -class ModelTest2(torch.nn.Module): +class ModelTestConv2(torch.nn.Module): """ Model to test. Principle features: @@ -132,7 +134,7 @@ def weight_init(module: torch.nn.Module): The module to initialize. """ if isinstance(module, torch.nn.Conv2d) or \ - isinstance(module, torch.nn.Linear): + isinstance(module, torch.nn.Linear): torch.nn.init.normal_(module.weight) if module.bias is not None: @@ -222,7 +224,7 @@ def _linear_decorrelate_color(x): return x -class ModelTest3(torch.nn.Module): +class ModelTestFFT(torch.nn.Module): """ Model to test. Principle features: @@ -261,7 +263,7 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return x -class ModelConv(torch.nn.Module): +class ModelTestConv(torch.nn.Module): """ Generic convolutional model to test. """ @@ -305,19 +307,27 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: return x -class ModelTest4(ModelConv): +class ModelTestConvSK(ModelTestConv): """ Model to test. Principle features: - - ConvTranspose2d with odd kernel and stride + - Conv2d + + Parameters + ---------- + stride: int + The stride of the model. + kernel: int + The kernel size of the model. """ - def __init__(self): + def __init__(self, stride: int, kernel: int): super().__init__() self.features = torch.nn.Sequential( - torch.nn.ConvTranspose2d( + torch.nn.Conv2d( 3, 5, - kernel_size=(3, 3), stride=(2, 2), + kernel_size=kernel, stride=stride, + padding=int(math.floor(kernel / 2)) ), ) self.avgpool = torch.nn.AdaptiveAvgPool2d((1, 1)) @@ -329,19 +339,26 @@ def __init__(self): self.classifier.apply(self.weight_init) -class ModelTest5(ModelConv): +class ModelTestDeConvSK(ModelTestConv): """ Model to test. Principle features: - - ConvTranspose2d with even kernel and stride + - ConvTranspose2d + + Parameters + ---------- + stride: int + The stride of the model. + kernel: int + The kernel size of the model. """ - def __init__(self): + def __init__(self, stride: int, kernel: int): super().__init__() self.features = torch.nn.Sequential( torch.nn.ConvTranspose2d( 3, 5, - kernel_size=(2, 2), stride=(2, 2), + kernel_size=kernel, stride=stride, ), ) self.avgpool = torch.nn.AdaptiveAvgPool2d((1, 1)) @@ -353,22 +370,179 @@ def __init__(self): self.classifier.apply(self.weight_init) -class ModelTest6(ModelConv): +class ModelTestCat(torch.nn.Module): """ Model to test. Principle features: - - ConvTranspose2d with odd kernel and no stride + - cat """ def __init__(self): super().__init__() + self.features1 = torch.nn.Sequential( + torch.nn.Conv2d( + 3, 6, + kernel_size=(1, 1), + bias=True + ), + ) + self.features2 = torch.nn.Sequential( + torch.nn.Conv2d( + 3, 9, + kernel_size=(1, 1), + bias=True + ), + ) + self.avgpool = torch.nn.AdaptiveAvgPool2d((1, 1)) + self.classifier = torch.nn.Sequential( + torch.nn.Linear(in_features=15, out_features=1) + ) + self.features1.apply(self.weight_init) + self.features2.apply(self.weight_init) + self.classifier.apply(self.weight_init) + + @staticmethod + def weight_init(module: torch.nn.Module): + """ + Initialize weights and biases. + + Parameters + ---------- + module: torch.nn.Module + The module to initialize. + """ + if isinstance(module, torch.nn.Conv2d) or \ + isinstance(module, torch.nn.Linear): + torch.nn.init.normal_(module.weight) + + if module.bias is not None: + torch.nn.init.normal_(module.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass. + + Parameters + ---------- + x: torch.Tensor + The input tensor. + + Returns + ------- + _: torch.Tensor + The output tensor. + """ + x1 = self.features1(x) + x2 = self.features2(x) + x = torch.cat([x1, x2], dim=1) + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.classifier(x) + return x + + +class ModelTestResize(torch.nn.Module): + """ + Model to test. + Principle features: + - resize + + Parameters + ---------- + size: int + The output size of the resize operation. + """ + + def __init__(self, size: int): + super().__init__() + self._size = size + self.features = torch.nn.Sequential( - torch.nn.ConvTranspose2d( + torch.nn.Conv2d( 3, 5, - kernel_size=(3, 3), stride=(1, 1), + kernel_size=(1, 1), + bias=True ), ) self.avgpool = torch.nn.AdaptiveAvgPool2d((1, 1)) + self.classifier = torch.nn.Sequential( + torch.nn.Linear(in_features=5, out_features=1) + ) + self.features.apply(self.weight_init) + self.classifier.apply(self.weight_init) + + @staticmethod + def weight_init(module: torch.nn.Module): + """ + Initialize weights and biases. + + Parameters + ---------- + module: torch.nn.Module + The module to initialize. + """ + if isinstance(module, torch.nn.Conv2d) or \ + isinstance(module, torch.nn.Linear): + torch.nn.init.normal_(module.weight) + + if module.bias is not None: + torch.nn.init.normal_(module.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass. + + Parameters + ---------- + x: torch.Tensor + The input tensor. + + Returns + ------- + _: torch.Tensor + The output tensor. + """ + x = self.features(x) + x = torch.nn.functional.interpolate( + x, + mode="bilinear", + size=self._size, + align_corners=True + ) + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.classifier(x) + return x + + +class ModelTestPatchConv(torch.nn.Module): + """ + Model to test. + Principle features: + - Patch convolution + + Parameters + ---------- + size: int + size of the input data. + patch: int + kernel split size of the input data. + """ + + def __init__(self, size: int, patch: int): + super().__init__() + + self._patch = patch + self._nb_patch = int(size / patch) + + self.features = torch.nn.Sequential( + torch.nn.Conv2d( + 3, 5, + kernel_size=(patch, patch), + stride=(patch, patch), + bias=True + ), + ) self.classifier = torch.nn.Sequential( torch.nn.Linear(in_features=5, out_features=1), ) @@ -376,23 +550,75 @@ def __init__(self): self.features.apply(self.weight_init) self.classifier.apply(self.weight_init) + @staticmethod + def weight_init(module: torch.nn.Module): + """ + Initialize weights and biases. -class ModelTest7(ModelConv): + Parameters + ---------- + module: torch.nn.Module + The module to initialize. + """ + if isinstance(module, torch.nn.Conv2d) or \ + isinstance(module, torch.nn.Linear): + torch.nn.init.normal_(module.weight) + + if module.bias is not None: + torch.nn.init.normal_(module.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass. + + Parameters + ---------- + x: torch.Tensor + The input tensor. + + Returns + ------- + _: torch.Tensor + The output tensor. + """ + x = self.features(x) + x = x.reshape(1, 5, self._nb_patch * self._nb_patch) + x = x.permute(0, 2, 1) + x = torch.mean(x, dim=1) + x = torch.flatten(x, 1) + x = self.classifier(x) + return x + + +class ModelTestAttention1(torch.nn.Module): """ Model to test. Principle features: - - ConvTranspose2d with even kernel and no stride + - MultiheadAttention with 1 head + + Parameters + ---------- + size: int + size of the input data. + patch: int + kernel split size of the input data. """ - def __init__(self): + def __init__(self, size: int, patch: int): super().__init__() + + self._patch = patch + self._nb_patch = int(size / patch) + self.features = torch.nn.Sequential( - torch.nn.ConvTranspose2d( + torch.nn.Conv2d( 3, 5, - kernel_size=(2, 2), stride=(1, 1), + kernel_size=(patch, patch), + stride=(patch, patch), + bias=True ), ) - self.avgpool = torch.nn.AdaptiveAvgPool2d((1, 1)) + self.attention = torch.nn.MultiheadAttention(5, 1, batch_first=True) self.classifier = torch.nn.Sequential( torch.nn.Linear(in_features=5, out_features=1), ) @@ -400,36 +626,169 @@ def __init__(self): self.features.apply(self.weight_init) self.classifier.apply(self.weight_init) + torch.nn.init.normal_(self.attention.in_proj_weight) + torch.nn.init.normal_(self.attention.in_proj_bias) + torch.nn.init.normal_(self.attention.out_proj.weight) + torch.nn.init.normal_(self.attention.out_proj.bias) + + @staticmethod + def weight_init(module: torch.nn.Module): + """ + Initialize weights and biases. + + Parameters + ---------- + module: torch.nn.Module + The module to initialize. + """ + if isinstance(module, torch.nn.Conv2d) or \ + isinstance(module, torch.nn.Linear): + torch.nn.init.normal_(module.weight) + + if module.bias is not None: + torch.nn.init.normal_(module.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass. -class ModelTest8(torch.nn.Module): + Parameters + ---------- + x: torch.Tensor + The input tensor. + + Returns + ------- + _: torch.Tensor + The output tensor. + """ + x = self.features(x) + x = x.reshape(1, 5, self._nb_patch * self._nb_patch) + x = x.permute(0, 2, 1) + x, _ = self.attention(x, x, x, need_weights=False) + x = torch.mean(x, dim=1) + x = torch.flatten(x, 1) + x = self.classifier(x) + return x + + +class ModelTestAttention2(torch.nn.Module): """ Model to test. Principle features: - - cat + - MultiheadAttention with 3 heads + + Parameters + ---------- + size: int + size of the input data. + patch: int + kernel split size of the input data. """ - def __init__(self): + def __init__(self, size: int, patch: int): super().__init__() - self.features1 = torch.nn.Sequential( + + self._patch = patch + self._nb_patch = int(size / patch) + + self.features = torch.nn.Sequential( torch.nn.Conv2d( 3, 6, - kernel_size=(1, 1), + kernel_size=(patch, patch), + stride=(patch, patch), bias=True ), ) - self.features2 = torch.nn.Sequential( + self.attention = torch.nn.MultiheadAttention(6, 3, batch_first=True) + self.classifier = torch.nn.Sequential( + torch.nn.Linear(in_features=6, out_features=1), + ) + + self.features.apply(self.weight_init) + self.classifier.apply(self.weight_init) + + torch.nn.init.normal_(self.attention.in_proj_weight) + torch.nn.init.normal_(self.attention.in_proj_bias) + torch.nn.init.normal_(self.attention.out_proj.weight) + torch.nn.init.normal_(self.attention.out_proj.bias) + + @staticmethod + def weight_init(module: torch.nn.Module): + """ + Initialize weights and biases. + + Parameters + ---------- + module: torch.nn.Module + The module to initialize. + """ + if isinstance(module, torch.nn.Conv2d) or \ + isinstance(module, torch.nn.Linear): + torch.nn.init.normal_(module.weight) + + if module.bias is not None: + torch.nn.init.normal_(module.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass. + + Parameters + ---------- + x: torch.Tensor + The input tensor. + + Returns + ------- + _: torch.Tensor + The output tensor. + """ + x = self.features(x) + x = x.reshape(1, 6, self._nb_patch * self._nb_patch) + x = x.permute(0, 2, 1) + x, _ = self.attention(x, x, x, need_weights=False) + x = torch.mean(x, dim=1) + x = torch.flatten(x, 1) + x = self.classifier(x) + return x + + +class ModelTestLayerNorm(torch.nn.Module): + """ + Model to test. + Principle features: + - LayerNorm + + Parameters + ---------- + size: int + size of the input data. + patch: int + kernel split size of the input data. + """ + + def __init__(self, size: int, patch: int): + super().__init__() + + self._patch = patch + self._nb_patch = int(size / patch) + + self.features = torch.nn.Sequential( torch.nn.Conv2d( - 3, 9, - kernel_size=(1, 1), + 3, 5, + kernel_size=(patch, patch), + stride=(patch, patch), bias=True ), ) - self.avgpool = torch.nn.AdaptiveAvgPool2d((1, 1)) + self.ln = torch.nn.LayerNorm(5) self.classifier = torch.nn.Sequential( - torch.nn.Linear(in_features=15, out_features=1) + torch.nn.Linear(in_features=5, out_features=1), ) - self.features1.apply(self.weight_init) - self.features2.apply(self.weight_init) + + self.features.apply(self.weight_init) + self.ln.apply(self.weight_init) self.classifier.apply(self.weight_init) @staticmethod @@ -449,6 +808,10 @@ def weight_init(module: torch.nn.Module): if module.bias is not None: torch.nn.init.normal_(module.bias) + elif isinstance(module, torch.nn.LayerNorm): + torch.nn.init.normal_(module.weight) + torch.nn.init.normal_(module.bias) + def forward(self, x: torch.Tensor) -> torch.Tensor: """ Forward pass. @@ -463,10 +826,131 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: _: torch.Tensor The output tensor. """ - x1 = self.features1(x) - x2 = self.features2(x) - x = torch.cat([x1, x2], dim=1) - x = self.avgpool(x) + x = self.features(x) + x = x.reshape(1, 5, self._nb_patch * self._nb_patch) + x = x.permute(0, 2, 1) + x = self.ln(x) + x = torch.mean(x, dim=1) x = torch.flatten(x, 1) x = self.classifier(x) return x + + +class ModelTestAutoEncoder(torch.nn.Module): + """ + Generic auto encoder model to test. + """ + + @staticmethod + def weight_init(module: torch.nn.Module): + """ + Initialize weights and biases. + + Parameters + ---------- + module: torch.nn.Module + The module to initialize. + """ + if isinstance(module, torch.nn.Conv2d) or \ + isinstance(module, torch.nn.ConvTranspose2d) or \ + isinstance(module, torch.nn.Linear): + torch.nn.init.normal_(module.weight) + + if module.bias is not None: + torch.nn.init.normal_(module.bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + Forward pass. + + Parameters + ---------- + x: torch.Tensor + The input tensor. + + Returns + ------- + _: torch.Tensor + The output tensor. + """ + x = self.features(x) + return x + + +class ModelTestAutoEncoder1(ModelTestAutoEncoder): + """ + Model to test. + Principle features: + - Conv2d + - ConvTranspose2d + """ + + def __init__(self) -> None: + super().__init__() + self.features = torch.nn.Sequential( + torch.nn.Conv2d(3, 5, kernel_size=3, stride=2, padding=1), + torch.nn.ConvTranspose2d(5, 3, kernel_size=2, stride=2), + ) + self.features.apply(self.weight_init) + + +class ModelTestGram(torch.nn.Module): + """ + Model to test. + Principle features: + - SelfCorrelate + - Normalize12 + - SimilarityBatchError + """ + + def __init__(self): + super().__init__() + self.features = torch.nn.Conv2d( + 3, 5, + kernel_size=1 + ) + self.avgpool = torch.nn.AdaptiveAvgPool2d((1, 1)) + self.classifier = torch.nn.Linear(in_features=5, out_features=1) + + self.features.apply(self.weight_init) + self.classifier.apply(self.weight_init) + + @staticmethod + def weight_init(module: torch.nn.Module): + """ + Initialize weights and biases. + + Parameters + ---------- + module: torch.nn.Module + The module to initialize. + """ + if isinstance(module, torch.nn.Conv2d) or \ + isinstance(module, torch.nn.Linear): + torch.nn.init.normal_(module.weight) + + if module.bias is not None: + torch.nn.init.normal_(module.bias) + + def forward(self, x: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Forward pass. + + Parameters + ---------- + x: torch.Tensor + The input tensor. + + Returns + ------- + _: (torch.Tensor, torch.Tensor) + The output tensor. + The features tensor. + """ + x = self.features(x) + features = x + + x = self.avgpool(x) + x = torch.flatten(x, 1) + x = self.classifier(x) + return x, features diff --git a/Tests/GrAITorchTests/Base/python_lib/weight.py b/Tests/GrAITorchTests/Base/python_lib/weight.py index 74920544..8be27013 100644 --- a/Tests/GrAITorchTests/Base/python_lib/weight.py +++ b/Tests/GrAITorchTests/Base/python_lib/weight.py @@ -3,13 +3,18 @@ from typing import List, Tuple from python_lib.model import ( - ModelTest1, - ModelTest2, - ModelTest4, - ModelTest5, - ModelTest6, - ModelTest7, - ModelTest8, + ModelTestConv1, + ModelTestConv2, + ModelTestConvSK, + ModelTestDeConvSK, + ModelTestCat, + ModelTestResize, + ModelTestPatchConv, + ModelTestAttention1, + ModelTestAttention2, + ModelTestLayerNorm, + ModelTestAutoEncoder1, + ModelTestGram, ) @@ -121,9 +126,102 @@ def _extract_and_transpose_weights( return layers_weights, layers_dims -def load_test1_weights() -> Tuple[List[List[float]], List[List[int]]]: +def _extract_attention_weights( + model: torch.nn.Module, +) -> Tuple[List[List[float]], List[List[int]]]: + """ + Get weights and biases. + + Parameters + ---------- + model: torch.nn.Module + The module to get the weights and biases from. + + Returns + ------- + (_, _): List[List[float]], List[List[int]] + The flattened weights, their shape. + """ + model_weights = model.state_dict() + + layers_weights: List[List[float]] = [] + layers_dims: List[List[int]] = [] + + cur_item = 0 + list_items = list(model_weights.items()) + + while cur_item < len(list_items): + name, layer_weights = list_items[cur_item] + print(f"Extracting weigths {name}.") + + if "in_proj" in name: + weights = layer_weights.data.cpu().numpy() + nb_partial = int(len(weights) / 3) + + weights1 = weights[0: nb_partial] + weights2 = weights[nb_partial: 2*nb_partial] + weights3 = weights[2*nb_partial: 3*nb_partial] + + cur_item += 1 + name, layer_weights = list_items[cur_item] + print(f"Extracting weigths {name}.") + biases = layer_weights.data.cpu().numpy() + + biases1 = biases[0: nb_partial] + biases2 = biases[nb_partial: 2 * nb_partial] + biases3 = biases[2 * nb_partial: 3 * nb_partial] + + weights_list, dims_list = _flatten_weights( + weights=weights1 + ) + layers_weights.append(weights_list) + layers_dims.append(dims_list) + weights_list, dims_list = _flatten_weights( + weights=biases1 + ) + layers_weights.append(weights_list) + layers_dims.append(dims_list) + + weights_list, dims_list = _flatten_weights( + weights=weights2 + ) + layers_weights.append(weights_list) + layers_dims.append(dims_list) + weights_list, dims_list = _flatten_weights( + weights=biases2 + ) + layers_weights.append(weights_list) + layers_dims.append(dims_list) + + weights_list, dims_list = _flatten_weights( + weights=weights3 + ) + layers_weights.append(weights_list) + layers_dims.append(dims_list) + weights_list, dims_list = _flatten_weights( + weights=biases3 + ) + layers_weights.append(weights_list) + layers_dims.append(dims_list) + + cur_item += 1 + + else: + weights_list, dims_list = _flatten_weights( + layer_weights.data.cpu().numpy() + ) + + layers_weights.append(weights_list) + layers_dims.append(dims_list) + + cur_item += 1 + + return layers_weights, layers_dims + + +def load_conv1_weights() -> Tuple[List[List[float]], List[List[int]]]: """ - Get weights and biases for ModelTest1. + Get weights and biases for ModelTestConv1. Returns ------- @@ -131,13 +229,13 @@ def load_test1_weights() -> Tuple[List[List[float]], List[List[int]]]: The flattened weights, their shape. """ torch.manual_seed(42) - model = ModelTest1() + model = ModelTestConv1() return _extract_weights(model) -def load_test2_weights() -> Tuple[List[List[float]], List[List[int]]]: +def load_conv2_weights() -> Tuple[List[List[float]], List[List[int]]]: """ - Get weights and biases for ModelTest2. + Get weights and biases for ModelTestConv2. Returns ------- @@ -145,13 +243,22 @@ def load_test2_weights() -> Tuple[List[List[float]], List[List[int]]]: The flattened weights, their shape. """ torch.manual_seed(42) - model = ModelTest2() + model = ModelTestConv2() return _extract_weights(model) -def load_test4_weights() -> Tuple[List[List[float]], List[List[int]]]: +def load_conv_sk_weights( + stride: int, kernel: int +) -> Tuple[List[List[float]], List[List[int]]]: """ - Get weights and biases for ModelTest4. + Get weights and biases for ModelTestConvSK. + + Parameters + ---------- + stride: int + The stride of the model. + kernel: int + The kernel size of the model. Returns ------- @@ -159,13 +266,22 @@ def load_test4_weights() -> Tuple[List[List[float]], List[List[int]]]: The flattened weights, their shape. """ torch.manual_seed(42) - model = ModelTest4() - return _extract_and_transpose_weights(list(model.children())) + model = ModelTestConvSK(stride=stride, kernel=kernel) + return _extract_weights(model) -def load_test5_weights() -> Tuple[List[List[float]], List[List[int]]]: +def load_deconv_sk_weights( + stride: int, kernel: int +) -> Tuple[List[List[float]], List[List[int]]]: """ - Get weights and biases for ModelTest5. + Get weights and biases for ModelTestDeConvSK. + + Parameters + ---------- + stride: int + The stride of the model. + kernel: int + The kernel size of the model. Returns ------- @@ -173,13 +289,13 @@ def load_test5_weights() -> Tuple[List[List[float]], List[List[int]]]: The flattened weights, their shape. """ torch.manual_seed(42) - model = ModelTest5() + model = ModelTestDeConvSK(stride=stride, kernel=kernel) return _extract_and_transpose_weights(list(model.children())) -def load_test6_weights() -> Tuple[List[List[float]], List[List[int]]]: +def load_cat_weights() -> Tuple[List[List[float]], List[List[int]]]: """ - Get weights and biases for ModelTest6. + Get weights and biases for ModelTestCat. Returns ------- @@ -187,13 +303,124 @@ def load_test6_weights() -> Tuple[List[List[float]], List[List[int]]]: The flattened weights, their shape. """ torch.manual_seed(42) - model = ModelTest6() - return _extract_and_transpose_weights(list(model.children())) + model = ModelTestCat() + return _extract_weights(model) + + +def load_resize_weights(size: int) -> Tuple[List[List[float]], List[List[int]]]: + """ + Get weights and biases for ModelTestResize. + + Parameters + ---------- + size: int + The output size of the resize operation. + + Returns + ------- + (_, _): List[float], List[int] + The flattened weights, their shape. + """ + torch.manual_seed(42) + model = ModelTestResize(size) + return _extract_weights(model) + + +def load_patch_conv_weights( + size: int, patch: int +) -> Tuple[List[List[float]], List[List[int]]]: + """ + Get weights and biases for ModelTestPatchConv. + + Parameters + ---------- + size: int + The size of the input data. + patch: int + kernel split size of the input data. + + Returns + ------- + (_, _): List[float], List[int] + The flattened weights, their shape. + """ + torch.manual_seed(42) + model = ModelTestPatchConv(size=size, patch=patch) + return _extract_weights(model) + + +def load_attention1_weights( + size: int, patch: int +) -> Tuple[List[List[float]], List[List[int]]]: + """ + Get weights and biases for ModelTestAttention1. + + Parameters + ---------- + size: int + The size of the input data. + patch: int + kernel split size of the input data. + + Returns + ------- + (_, _): List[float], List[int] + The flattened weights, their shape. + """ + torch.manual_seed(42) + model = ModelTestAttention1(size=size, patch=patch) + return _extract_attention_weights(model=model) + + +def load_attention2_weights( + size: int, patch: int +) -> Tuple[List[List[float]], List[List[int]]]: + """ + Get weights and biases for ModelTestAttention2. + + Parameters + ---------- + size: int + The size of the input data. + patch: int + kernel split size of the input data. + + Returns + ------- + (_, _): List[float], List[int] + The flattened weights, their shape. + """ + torch.manual_seed(42) + model = ModelTestAttention2(size=size, patch=patch) + return _extract_attention_weights(model=model) + + +def load_layer_norm_weights( + size: int, patch: int +) -> Tuple[List[List[float]], List[List[int]]]: + """ + Get weights and biases for ModelTestLayerNorm. + + Parameters + ---------- + size: int + The size of the input data. + patch: int + kernel split size of the input data. + + Returns + ------- + (_, _): List[float], List[int] + The flattened weights, their shape. + """ + torch.manual_seed(42) + model = ModelTestLayerNorm(size=size, patch=patch) + return _extract_weights(model) -def load_test7_weights() -> Tuple[List[List[float]], List[List[int]]]: +def load_auto_encoder1_weights() -> Tuple[List[List[float]], List[List[int]]]: """ - Get weights and biases for ModelTest7. + Get weights and biases for ModelTestAutoEncoder1. Returns ------- @@ -201,13 +428,13 @@ def load_test7_weights() -> Tuple[List[List[float]], List[List[int]]]: The flattened weights, their shape. """ torch.manual_seed(42) - model = ModelTest7() + model = ModelTestAutoEncoder1() return _extract_and_transpose_weights(list(model.children())) -def load_test8_weights() -> Tuple[List[List[float]], List[List[int]]]: +def load_gram_weights() -> Tuple[List[List[float]], List[List[int]]]: """ - Get weights and biases for ModelTest8. + Get weights and biases for ModelTestGram. Returns ------- @@ -215,5 +442,5 @@ def load_test8_weights() -> Tuple[List[List[float]], List[List[int]]]: The flattened weights, their shape. """ torch.manual_seed(42) - model = ModelTest8() + model = ModelTestGram() return _extract_weights(model) diff --git a/Tests/GrAITorchTests/GrAITorchTests.swift b/Tests/GrAITorchTests/GrAITorchTests.swift index dbd73cbe..1454cba8 100644 --- a/Tests/GrAITorchTests/GrAITorchTests.swift +++ b/Tests/GrAITorchTests/GrAITorchTests.swift @@ -8,11 +8,13 @@ import XCTest import GrAIdient -/// Compare models created by GrAIdient and PyTorch. +/// Compare models created in GrAIdient and PyTorch. final class GrAITorchTests: XCTestCase { /// Size of one image (height and width are the same). let _size = 32 + /// Kernel split size of one image (height and width are the same). + let _patch = 8 /// Initialize test. override func setUp() @@ -25,10 +27,12 @@ final class GrAITorchTests: XCTestCase /// /// Compute the gradient norm on the first layer of the model. /// - /// - Parameter model: The model we want to evalulate the gradient norm on. + /// - Parameters: + /// - model: The model we want to evalulate the gradient norm on. + /// - size: The size of the input data. /// - Returns: The gradient norm on the first layer. /// - func _getGradientNorm(_ model: Model) -> Double + func _getGradientNormMSE1D(model: Model, size: Int) -> Double { // Create the context to build a graph of layers // that come after the layers inside `model`. @@ -43,18 +47,31 @@ final class GrAITorchTests: XCTestCase lastLayer.coeff = 1.0 / 2.0 // Initialize the finalModel with the links (`layerPrev` updated). - let finalModel = Model(model: context.model, modelsPrev: [model]) + context.model.layers = model.layers + context.model.layers + let finalModel = Model(model: context.model, modelsPrev: []) // Initialize for inference. finalModel.initKernel(phase: .Inference) - // The final model contains the layers of `model` and the loss layer. - finalModel.layers = model.layers + context.model.layers + // Avoid the compute of every gradients of weights. + model.computeDeltaWeights = false let optimizerParams = getOptimizerParams(nbLoops: 1) finalModel.setupOptimizers(params: optimizerParams) - let groundTruth: [[Double]] = [[0.0]] - let firstLayer: Input2D = finalModel.layers.first as! Input2D + let firstLayer: Input2D = model.layers.first as! Input2D + // Allow backward pass go through the first layer. + firstLayer.computeDelta = true + // Allow to compute the gradients of weights for the first layer. + firstLayer.computeDeltaWeights = true + + // Set data. + let data: [Float] = getInputData(size) + try! firstLayer.setDataGPU( + data, + batchSize: 1, + nbChannels: 3, height: _size, width: _size, + format: .RGB + ) // Update internal batch size. finalModel.updateKernel(batchSize: 1) @@ -63,7 +80,12 @@ final class GrAITorchTests: XCTestCase try! finalModel.forward() // Apply loss derivative. - try! lastLayer.lossDerivativeGPU(groundTruth) + let groundTruth: [[Double]] = [[0.0]] + try! lastLayer.lossDerivativeGPU( + groundTruth, + batchSize: 1, + nbNeurons: 1 + ) // Backward. try! finalModel.backward() @@ -74,17 +96,39 @@ final class GrAITorchTests: XCTestCase return gradNormOutput } - /// Test that model1 backward pass returns the same gradient norm in GrAIdient and PyTorch. - func testModel1() + /// + /// Compute the gradient norm on the first layer of the model. + /// + /// - Parameters: + /// - model: The model we want to evalulate the gradient norm on. + /// - size: The size of the input data. + /// - Returns: The gradient norm on the first layer. + /// + func _getGradientNormMSE2D(model: Model, size: Int) -> Double { - // Build model. - let model = ModelTest1.build(_size) + // Create the context to build a graph of layers + // that come after the layers inside `model`. + let context = ModelContext(name: "ModelTest", models: [model]) + let params = GrAI.Model.Params(context: context) + + // Append a loss layer. + let lastLayer = try! MSE2D( + layerPrev: model.layers.last! as! Layer2D, + params: params + ) + + // Initialize the finalModel with the links (`layerPrev` updated). + context.model.layers = model.layers + context.model.layers + let finalModel = Model(model: context.model, modelsPrev: [model]) // Initialize for inference. - model.initKernel(phase: .Inference) + finalModel.initKernel(phase: .Inference) // Avoid the compute of every gradients of weights. model.computeDeltaWeights = false + let optimizerParams = getOptimizerParams(nbLoops: 1) + finalModel.setupOptimizers(params: optimizerParams) + let firstLayer: Input2D = model.layers.first as! Input2D // Allow backward pass go through the first layer. firstLayer.computeDelta = true @@ -92,30 +136,87 @@ final class GrAITorchTests: XCTestCase firstLayer.computeDeltaWeights = true // Set data. - let data: [Float] = getInputData(_size) - try! firstLayer.setDataGPU(data, batchSize: 1, format: .RGB) + let data: [Float] = getInputData(size) + try! firstLayer.setDataGPU( + data, + batchSize: 1, + nbChannels: 3, height: _size, width: _size, + format: .RGB + ) - // Get the gradient norm on the first layer. - let expectedNorm: Double = Double(computeTest1GradNorm(_size)) - let gradNormOutput: Double = _getGradientNorm(model) + // Update internal batch size. + finalModel.updateKernel(batchSize: 1) - // Compare difference. - let diffPercent = - abs(gradNormOutput - expectedNorm) / expectedNorm * 100.0 - XCTAssert(diffPercent < 1.0) + // Forward. + try! finalModel.forward() + + // Apply loss derivative. + try! lastLayer.lossDerivativeGPU( + data, + batchSize: 1, + nbChannels: 3, height: _size, width: _size, + format: .RGB + ) + + // Backward. + try! finalModel.backward() + + // Get the gradient norm on the first layer. + let gradNormOutput: Double = + try! finalModel.getGradientNorm(layers: [firstLayer]) + return gradNormOutput } - /// Test that model2 backward pass returns the same gradient norm in GrAIdient and PyTorch. - func testModel2() + /// + /// Compute the gradient norm on the first layer of the model. + /// + /// - Parameters: + /// - model: The model we want to evalulate the gradient norm on. + /// - size: The size of the input data. + /// - batchSize: The number of images in the batch. + /// - Returns: The gradient norm on the first layer. + /// + func _getGradientNormGram2D( + model: Model, + size: Int, + batchSize: Int) -> Double { - // Build model. - let model = ModelTest2.build(_size) + // Create the context to build a graph of layers + // that come after the layers inside `model`. + let context = ModelContext(name: "ModelTest", models: [model]) + let params = GrAI.Model.Params(context: context) + + var layer: Layer2D = SelfCorrelate2D( + layerPrev: model.layers[1] as! Layer2D, + params: params + ) + + layer = Normalize122D(layerPrev: layer, params: params) + + let similarityLayer = try! SimilarityBatchError2D( + layerPrev: layer, + params: params + ) + + // Append a loss layer. + let lastLayer = MSE1D( + layerPrev: model.layers.last! as! Layer1D, + params: params + ) + lastLayer.coeff = 1.0 / 2.0 + + // Initialize the finalModel with the links (`layerPrev` updated). + context.model.layers = model.layers + context.model.layers + let finalModel = Model(model: context.model, modelsPrev: []) // Initialize for inference. - model.initKernel(phase: .Inference) + finalModel.initKernel(phase: .Inference) // Avoid the compute of every gradients of weights. model.computeDeltaWeights = false + let optimizerParams = getOptimizerParams(nbLoops: 1) + finalModel.setupOptimizers(params: optimizerParams) + let firstLayer: Input2D = model.layers.first as! Input2D // Allow backward pass go through the first layer. firstLayer.computeDelta = true @@ -123,12 +224,71 @@ final class GrAITorchTests: XCTestCase firstLayer.computeDeltaWeights = true // Set data. - let data: [Float] = getInputData(_size) - try! firstLayer.setDataGPU(data, batchSize: 1, format: .RGB) + let data: [Float] = getBatchData(size: size, batchSize: batchSize) + try! firstLayer.setDataGPU( + data, + batchSize: batchSize, + nbChannels: 3, height: _size, width: _size, + format: .RGB + ) + + // Update internal batch size. + finalModel.updateKernel(batchSize: batchSize) + + // Forward. + try! finalModel.forward() + + // Apply loss derivative. + var groundTruth = [[Double]]() + for _ in 0..