diff --git a/CMakeLists.txt b/CMakeLists.txt index fcd93915..667b5d73 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -64,10 +64,10 @@ add_library(neural-fortran src/nf/nf_parallel.f90 src/nf/nf_parallel_submodule.f90 src/nf/nf_random.f90 - src/nf/nf_reshape_layer.f90 - src/nf/nf_reshape_layer_submodule.f90 src/nf/nf_reshape2d_layer.f90 src/nf/nf_reshape2d_layer_submodule.f90 + src/nf/nf_reshape3d_layer.f90 + src/nf/nf_reshape3d_layer_submodule.f90 src/nf/nf_self_attention_layer.f90 src/nf/io/nf_io_binary.f90 src/nf/io/nf_io_binary_submodule.f90 diff --git a/README.md b/README.md index 75da6525..65964786 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,8 @@ Read the paper [here](https://arxiv.org/abs/1902.06714). ## Features -* Training and inference of dense (fully connected) and convolutional neural - networks +* Training and inference of dense (fully connected), convolutional (1-d and 2-d), + and transformer neural networks * Stochastic gradient descent optimizers: Classic, momentum, Nesterov momentum, RMSProp, Adagrad, Adam, AdamW * More than a dozen activation functions and their derivatives @@ -41,9 +41,8 @@ Read the paper [here](https://arxiv.org/abs/1902.06714). | Linear (2-d) | `linear2d` | `input2d`, `layernorm`, `linear2d`, `self_attention` | 2 | ✅ | ✅ | | Self-attention | `self_attention` | `input2d`, `layernorm`, `linear2d`, `self_attention` | 2 | ✅ | ✅ | | Layer Normalization | `layernorm` | `linear2d`, `self_attention` | 2 | ✅ | ✅ | -| Flatten | `flatten` | `input2d`, `input3d`, `conv2d`, `maxpool2d`, `reshape` | 1 | ✅ | ✅ | -| Reshape (1-d to 2-d) | `reshape2d` | `input2d`, `conv1d`, `locally_connected1d`, `maxpool1d` | 2 | ✅ | ✅ | -| Reshape (1-d to 3-d) | `reshape` | `input1d`, `dense`, `flatten` | 3 | ✅ | ✅ | +| Flatten | `flatten` | `input2d`, `input3d`, `conv1d`, `conv2d`, `maxpool1d`, `maxpool2d`, `reshape` | 1 | ✅ | ✅ | +| Reshape (1-d to 2-d or 3-d) | `reshape` | `dense`, `dropout`, `flatten`, `input1d` | 2, 3 | ✅ | ✅ | ## Getting started @@ -263,11 +262,13 @@ It may be useful to read if you want to contribute a new feature to neural-fortr Thanks to all open-source contributors to neural-fortran: [awvwgk](https://github.com/awvwgk), +[certik](https://github.com/certik), [ggoyman](https://github.com/ggoyman), [ivan-pi](https://github.com/ivan-pi), [jacobwilliams](https://github.com/jacobwilliams), [jvdp1](https://github.com/jvdp1), [jvo203](https://github.com/jvo203), +[mathomp4](https://github.com/mathomp4), [milancurcic](https://github.com/milancurcic), [OneAdder](https://github.com/OneAdder), [pirpyn](https://github.com/pirpyn), diff --git a/example/cnn_mnist.f90 b/example/cnn_mnist.f90 index ef22f986..d2f61723 100644 --- a/example/cnn_mnist.f90 +++ b/example/cnn_mnist.f90 @@ -20,7 +20,7 @@ program cnn_mnist net = network([ & input(784), & - reshape([1,28,28]), & + reshape(1, 28, 28), & conv2d(filters=8, kernel_size=3, activation=relu()), & maxpool2d(pool_size=2), & conv2d(filters=16, kernel_size=3, activation=relu()), & diff --git a/example/cnn_mnist_1d.f90 b/example/cnn_mnist_1d.f90 index 7e978034..b350a2f0 100644 --- a/example/cnn_mnist_1d.f90 +++ b/example/cnn_mnist_1d.f90 @@ -1,7 +1,7 @@ program cnn_mnist_1d use nf, only: network, sgd, & - input, conv1d, maxpool1d, flatten, dense, reshape, reshape2d, locally_connected1d, & + input, conv1d, maxpool1d, flatten, dense, reshape, locally_connected1d, & load_mnist, label_digits, softmax, relu implicit none @@ -20,7 +20,7 @@ program cnn_mnist_1d net = network([ & input(784), & - reshape2d([28, 28]), & + reshape(28, 28), & locally_connected1d(filters=8, kernel_size=3, activation=relu()), & maxpool1d(pool_size=2), & locally_connected1d(filters=16, kernel_size=3, activation=relu()), & diff --git a/fpm.toml b/fpm.toml index 15a746e4..1f2c2ac9 100644 --- a/fpm.toml +++ b/fpm.toml @@ -1,5 +1,5 @@ name = "neural-fortran" -version = "0.20.0" +version = "0.21.0" license = "MIT" author = "Milan Curcic" maintainer = "mcurcic@miami.edu" diff --git a/src/nf.f90 b/src/nf.f90 index 172eafb3..f644826d 100644 --- a/src/nf.f90 +++ b/src/nf.f90 @@ -16,7 +16,6 @@ module nf maxpool1d, & maxpool2d, & reshape, & - reshape2d, & self_attention use nf_loss, only: mse, quadratic use nf_metrics, only: corr, maxabs diff --git a/src/nf/nf_layer_constructors.f90 b/src/nf/nf_layer_constructors.f90 index e5f92f64..d3f06ca3 100644 --- a/src/nf/nf_layer_constructors.f90 +++ b/src/nf/nf_layer_constructors.f90 @@ -20,7 +20,6 @@ module nf_layer_constructors maxpool1d, & maxpool2d, & reshape, & - reshape2d, & self_attention, & embedding, & layernorm @@ -94,6 +93,28 @@ end function input3d end interface input + + interface reshape + + module function reshape2d(dim1, dim2) result(res) + !! Rank-1 to rank-2 reshape layer constructor. + integer, intent(in) :: dim1, dim2 + !! Shape of the output + type(layer) :: res + !! Resulting layer instance + end function reshape2d + + module function reshape3d(dim1, dim2, dim3) result(res) + !! Rank-1 to rank-3 reshape layer constructor. + integer, intent(in) :: dim1, dim2, dim3 + !! Shape of the output + type(layer) :: res + !! Resulting layer instance + end function reshape3d + + end interface reshape + + interface module function dense(layer_size, activation) result(res) @@ -283,28 +304,6 @@ module function maxpool2d(pool_size, stride) result(res) !! Resulting layer instance end function maxpool2d - module function reshape(output_shape) result(res) - !! Rank-1 to rank-any reshape layer constructor. - !! Currently implemented is only rank-3 for the output of the reshape. - !! - !! This layer is for connecting 1-d inputs to conv2d or similar layers. - integer, intent(in) :: output_shape(:) - !! Shape of the output - type(layer) :: res - !! Resulting layer instance - end function reshape - - module function reshape2d(output_shape) result(res) - !! Rank-1 to rank-any reshape layer constructor. - !! Currently implemented is only rank-2 for the output of the reshape. - !! - !! This layer is for connecting 1-d inputs to conv1d or similar layers. - integer, intent(in) :: output_shape(:) - !! Shape of the output - type(layer) :: res - !! Resulting layer instance - end function reshape2d - module function linear2d(out_features) result(res) !! Rank-2 (sequence_length, out_features) linear layer constructor. !! sequence_length is determined at layer initialization, based on the diff --git a/src/nf/nf_layer_constructors_submodule.f90 b/src/nf/nf_layer_constructors_submodule.f90 index 48fcd8a5..1665d38a 100644 --- a/src/nf/nf_layer_constructors_submodule.f90 +++ b/src/nf/nf_layer_constructors_submodule.f90 @@ -12,8 +12,8 @@ use nf_locally_connected1d_layer, only: locally_connected1d_layer use nf_maxpool1d_layer, only: maxpool1d_layer use nf_maxpool2d_layer, only: maxpool2d_layer - use nf_reshape_layer, only: reshape3d_layer use nf_reshape2d_layer, only: reshape2d_layer + use nf_reshape3d_layer, only: reshape3d_layer use nf_linear2d_layer, only: linear2d_layer use nf_self_attention_layer, only: self_attention_layer use nf_embedding_layer, only: embedding_layer @@ -229,35 +229,22 @@ module function maxpool2d(pool_size, stride) result(res) end function maxpool2d - module function reshape(output_shape) result(res) - integer, intent(in) :: output_shape(:) - type(layer) :: res - - res % name = 'reshape' - res % layer_shape = output_shape - - if (size(output_shape) == 3) then - allocate(res % p, source=reshape3d_layer(output_shape)) - else - error stop 'size(output_shape) of the reshape layer must == 3' - end if - - end function reshape - - module function reshape2d(output_shape) result(res) - integer, intent(in) :: output_shape(:) + module function reshape2d(dim1, dim2) result(res) + integer, intent(in) :: dim1, dim2 type(layer) :: res - res % name = 'reshape2d' - res % layer_shape = output_shape + res % layer_shape = [dim1, dim2] + allocate(res % p, source=reshape2d_layer(res % layer_shape)) + end function reshape2d - if (size(output_shape) == 2) then - allocate(res % p, source=reshape2d_layer(output_shape)) - else - error stop 'size(output_shape) of the reshape layer must == 2' - end if - end function reshape2d + module function reshape3d(dim1, dim2, dim3) result(res) + integer, intent(in) :: dim1, dim2, dim3 + type(layer) :: res + res % name = 'reshape3d' + res % layer_shape = [dim1, dim2, dim3] + allocate(res % p, source=reshape3d_layer(res % layer_shape)) + end function reshape3d module function linear2d(out_features) result(res) diff --git a/src/nf/nf_layer_submodule.f90 b/src/nf/nf_layer_submodule.f90 index 63af7264..eebedaa9 100644 --- a/src/nf/nf_layer_submodule.f90 +++ b/src/nf/nf_layer_submodule.f90 @@ -13,7 +13,7 @@ use nf_maxpool1d_layer, only: maxpool1d_layer use nf_maxpool2d_layer, only: maxpool2d_layer use nf_reshape2d_layer, only: reshape2d_layer - use nf_reshape_layer, only: reshape3d_layer + use nf_reshape3d_layer, only: reshape3d_layer use nf_linear2d_layer, only: linear2d_layer use nf_self_attention_layer, only: self_attention_layer use nf_embedding_layer, only: embedding_layer diff --git a/src/nf/nf_network_submodule.f90 b/src/nf/nf_network_submodule.f90 index 1752f1f8..449b5a5b 100644 --- a/src/nf/nf_network_submodule.f90 +++ b/src/nf/nf_network_submodule.f90 @@ -12,13 +12,13 @@ use nf_maxpool1d_layer, only: maxpool1d_layer use nf_maxpool2d_layer, only: maxpool2d_layer use nf_reshape2d_layer, only: reshape2d_layer - use nf_reshape_layer, only: reshape3d_layer + use nf_reshape3d_layer, only: reshape3d_layer use nf_linear2d_layer, only: linear2d_layer use nf_self_attention_layer, only: self_attention_layer use nf_embedding_layer, only: embedding_layer use nf_layernorm_layer, only: layernorm_layer use nf_layer, only: layer - use nf_layer_constructors, only: conv1d, conv2d, dense, flatten, input, maxpool1d, maxpool2d, reshape, reshape2d + use nf_layer_constructors, only: conv1d, conv2d, dense, flatten, input, maxpool1d, maxpool2d, reshape use nf_loss, only: quadratic use nf_optimizers, only: optimizer_base_type, sgd use nf_parallel, only: tile_indices diff --git a/src/nf/nf_reshape_layer.f90 b/src/nf/nf_reshape3d_layer.f90 similarity index 97% rename from src/nf/nf_reshape_layer.f90 rename to src/nf/nf_reshape3d_layer.f90 index f17620e6..355bbfe0 100644 --- a/src/nf/nf_reshape_layer.f90 +++ b/src/nf/nf_reshape3d_layer.f90 @@ -1,4 +1,4 @@ -module nf_reshape_layer +module nf_reshape3d_layer !! This module provides the concrete reshape layer type. !! It is used internally by the layer type. @@ -73,4 +73,4 @@ end subroutine init end interface -end module nf_reshape_layer +end module nf_reshape3d_layer diff --git a/src/nf/nf_reshape_layer_submodule.f90 b/src/nf/nf_reshape3d_layer_submodule.f90 similarity index 92% rename from src/nf/nf_reshape_layer_submodule.f90 rename to src/nf/nf_reshape3d_layer_submodule.f90 index 989497ba..a867a59c 100644 --- a/src/nf/nf_reshape_layer_submodule.f90 +++ b/src/nf/nf_reshape3d_layer_submodule.f90 @@ -1,4 +1,4 @@ -submodule(nf_reshape_layer) nf_reshape_layer_submodule +submodule(nf_reshape3d_layer) nf_reshape3d_layer_submodule use nf_base_layer, only: base_layer @@ -48,4 +48,4 @@ module subroutine init(self, input_shape) end subroutine init -end submodule nf_reshape_layer_submodule +end submodule nf_reshape3d_layer_submodule diff --git a/test/test_insert_flatten.f90 b/test/test_insert_flatten.f90 index c6b64f80..18e41b81 100644 --- a/test/test_insert_flatten.f90 +++ b/test/test_insert_flatten.f90 @@ -45,7 +45,7 @@ program test_insert_flatten net = network([ & input(4), & - reshape([1, 2, 2]), & + reshape(1, 2, 2), & dense(4) & ]) diff --git a/test/test_reshape2d_layer.f90 b/test/test_reshape2d_layer.f90 index 52817eac..b80299bd 100644 --- a/test/test_reshape2d_layer.f90 +++ b/test/test_reshape2d_layer.f90 @@ -1,8 +1,7 @@ program test_reshape2d_layer use iso_fortran_env, only: stderr => error_unit - use nf, only: input, network, reshape2d_layer => reshape2d - use nf_datasets, only: download_and_unpack, keras_reshape_url + use nf, only: input, network, reshape2d => reshape implicit none @@ -10,14 +9,13 @@ program test_reshape2d_layer real, allocatable :: sample_input(:), output(:,:) integer, parameter :: output_shape(2) = [4,4] integer, parameter :: input_size = product(output_shape) - character(*), parameter :: keras_reshape_path = 'keras_reshape.h5' logical :: file_exists logical :: ok = .true. ! Create the network net = network([ & input(input_size), & - reshape2d_layer(output_shape) & + reshape2d(output_shape(1), output_shape(2)) & ]) if (.not. size(net % layers) == 2) then diff --git a/test/test_reshape_layer.f90 b/test/test_reshape_layer.f90 index a448fc12..707532d8 100644 --- a/test/test_reshape_layer.f90 +++ b/test/test_reshape_layer.f90 @@ -1,7 +1,7 @@ program test_reshape_layer use iso_fortran_env, only: stderr => error_unit - use nf, only: input, network, reshape_layer => reshape + use nf, only: input, network, reshape3d => reshape use nf_datasets, only: download_and_unpack, keras_reshape_url implicit none @@ -17,7 +17,7 @@ program test_reshape_layer ! Create the network net = network([ & input(input_size), & - reshape_layer(output_shape) & + reshape3d(3, 32, 32) & ]) if (.not. size(net % layers) == 2) then