From aaf5036b4381049feb1da8a289f658316814e860 Mon Sep 17 00:00:00 2001 From: Dobiasd Date: Thu, 13 Feb 2020 15:57:27 +0100 Subject: [PATCH 1/7] Update TensorFlow to version 2.1.0 --- .travis.yml | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4728659..7378c56b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ install: - sudo apt-get install libblas-dev liblapack-dev libatlas-base-dev gfortran # python libs - sudo pip3 install --upgrade pip - - sudo pip3 install numpy scipy h5py "tensorflow==2.0.0" + - sudo pip3 install numpy scipy h5py "tensorflow==2.1.0" - echo "Version numbers of TensorFlow and Keras:" - python3 -c "import tensorflow as tf; import tensorflow.keras; print(tf.__version__); print(tensorflow.keras.__version__)" # FunctionalPlus diff --git a/README.md b/README.md index 09aa8f16..7eb2d613 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ Requirements and Installation - A **C++14**-compatible compiler: Compilers from these versions on are fine: GCC 4.9, Clang 3.7 (libc++ 3.7) and Visual C++ 2015. - Python 3.5 or higher. -- TensorFlow 2.0.0 +- TensorFlow 2.1.0 Guides for different ways to install frugally-deep can be found in [`INSTALL.md`](INSTALL.md). From 52fc9082c62aef7ae6a28a980fb0d15da71dcde7 Mon Sep 17 00:00:00 2001 From: Keith Chugg Date: Wed, 8 Apr 2020 00:04:42 +0000 Subject: [PATCH 2/7] temp-files for TF 2.1 --- test/stateful_test/temp_rnn_results.txt | 58 ++++++++ test/stateful_test/temp_rnn_test.cpp | 57 ++++++++ test/stateful_test/temp_rnn_test.py | 68 +++++++++ test/stateful_test/temp_stateful.h5 | Bin 0 -> 13344 bytes test/stateful_test/temp_stateful.json | 184 ++++++++++++++++++++++++ test/stateful_test/temp_stateless.h5 | Bin 0 -> 13344 bytes test/stateful_test/temp_stateless.json | 184 ++++++++++++++++++++++++ 7 files changed, 551 insertions(+) create mode 100644 test/stateful_test/temp_rnn_results.txt create mode 100644 test/stateful_test/temp_rnn_test.cpp create mode 100644 test/stateful_test/temp_rnn_test.py create mode 100644 test/stateful_test/temp_stateful.h5 create mode 100644 test/stateful_test/temp_stateful.json create mode 100644 test/stateful_test/temp_stateless.h5 create mode 100644 test/stateful_test/temp_stateless.json diff --git a/test/stateful_test/temp_rnn_results.txt b/test/stateful_test/temp_rnn_results.txt new file mode 100644 index 00000000..a2d01f79 --- /dev/null +++ b/test/stateful_test/temp_rnn_results.txt @@ -0,0 +1,58 @@ + +TF 2.0: +----------------------- +non_stateful: [-7.676248 -7.5144324 -7.3429837] +stateful: [-7.676248 -7.5144324 -7.3429837] +delta: [0. 0. 0.] +non_stateful: [-7.676248 -7.5144324 -7.3429837] +stateful: [-7.676248 -7.5144324 -7.3429837] +delta: [0. 0. 0.] + +** RESETING STATES in STATEFUL MODEL ** + +non_stateful: [-7.676248 -7.5144324 -7.3429837] +stateful: [-7.676248 -7.5144324 -7.3429837] +delta: [0. 0. 0.] +non_stateful: [-7.676248 -7.5144324 -7.3429837] +stateful: [-7.676248 -7.5144324 -7.3429837] +delta: [0. 0. 0.] + + +TF 2.1: +----------------------- +non_stateful: [-7.676248 -7.5144324 -7.3429837] +stateful: [-7.676248 -7.5144324 -7.3429837] +delta: [0. 0. 0.] +non_stateful: [-7.676248 -7.5144324 -7.3429837] +stateful: [-7.247089 -7.058707 -6.857616] +delta: [0.42915916 0.4557252 0.48536777] + +** RESETING STATES in STATEFUL MODEL ** + +non_stateful: [-7.676248 -7.5144324 -7.3429837] +stateful: [-7.676248 -7.5144324 -7.3429837] +delta: [0. 0. 0.] +non_stateful: [-7.676248 -7.5144324 -7.3429837] +stateful: [-7.247089 -7.058707 -6.857616] +delta: [0.42915916 0.4557252 0.48536777] + +FD starting point: +----------------------- +starting A +Non-Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Non-Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +** RESETING STATES in STATEFUL MODEL ** +Non-Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Non-Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] \ No newline at end of file diff --git a/test/stateful_test/temp_rnn_test.cpp b/test/stateful_test/temp_rnn_test.cpp new file mode 100644 index 00000000..732558d8 --- /dev/null +++ b/test/stateful_test/temp_rnn_test.cpp @@ -0,0 +1,57 @@ +#include "fdeep/fdeep.hpp" +#include // looks like we need this too (edit by π) + +using namespace fdeep; + +int main() +{ + const std::vector x_inf_0 = {1.0, 0.0, 0.0}; + const std::vector state_0 = {10.0}; + + const shared_float_vec xt0(fplus::make_shared_ref(x_inf_0)); + const shared_float_vec st0(fplus::make_shared_ref(state_0)); + + std::cout << "convert to tensors" << std::endl; + const tensor test_in_0(tensor_shape(3, 1), xt0); + std::cout << "convert to tensors" << std::endl; + const tensor test_state_0(tensor_shape(static_cast(1)), st0); + + std::cout << "loading models" << std::endl; + auto stateful_model = load_model("temp_stateful.json"); + auto stateless_model = load_model("temp_stateless.json"); + + // A + std::cout << "starting A" << std::endl; + auto non_stateful_out = stateless_model.predict({test_in_0, test_state_0}); + auto stateful_out = stateful_model.predict_stateful({test_in_0, test_state_0}); + std::cout << "Non-Stateful" << std::endl; + std::cout << fdeep::show_tensors(non_stateful_out) << std::endl; + std::cout << "Stateful" << std::endl; + std::cout << fdeep::show_tensors(stateful_out) << std::endl; + + // B + non_stateful_out = stateless_model.predict({test_in_0, test_state_0}); + stateful_out = stateful_model.predict_stateful({test_in_0, test_state_0}); + std::cout << "Non-Stateful" << std::endl; + std::cout << fdeep::show_tensors(non_stateful_out) << std::endl; + std::cout << "Stateful" << std::endl; + std::cout << fdeep::show_tensors(stateful_out) << std::endl; + + // C + std::cout << "** RESETING STATES in STATEFUL MODEL **" << std::endl; + stateful_model.reset_states(); + non_stateful_out = stateless_model.predict({test_in_0, test_state_0}); + stateful_out = stateful_model.predict_stateful({test_in_0, test_state_0}); + std::cout << "Non-Stateful" << std::endl; + std::cout << fdeep::show_tensors(non_stateful_out) << std::endl; + std::cout << "Stateful" << std::endl; + std::cout << fdeep::show_tensors(stateful_out) << std::endl; + + //D + non_stateful_out = stateless_model.predict({test_in_0, test_state_0}); + stateful_out = stateful_model.predict_stateful({test_in_0, test_state_0}); + std::cout << "Non-Stateful" << std::endl; + std::cout << fdeep::show_tensors(non_stateful_out) << std::endl; + std::cout << "Stateful" << std::endl; + std::cout << fdeep::show_tensors(stateful_out) << std::endl; +} \ No newline at end of file diff --git a/test/stateful_test/temp_rnn_test.py b/test/stateful_test/temp_rnn_test.py new file mode 100644 index 00000000..acf9dedf --- /dev/null +++ b/test/stateful_test/temp_rnn_test.py @@ -0,0 +1,68 @@ +import os +os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' +os.environ['CUDA_VISIBLE_DEVICES']='' + +import numpy as np +from tensorflow.keras.layers import Input, Dense, SimpleRNN, GRU, LSTM, Bidirectional +from tensorflow.keras.models import Model + +USE_TOY_WEIGHTS = True +REC_LAYER = GRU +sequence_length = 3 +feature_dim = 1 +features_in = Input(batch_shape=(1, sequence_length, feature_dim)) +state_h_in = Input(batch_shape=(1, 1)) + +rnn_out = REC_LAYER(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=False)(features_in, initial_state=state_h_in) +stateless_model = Model(inputs=[features_in, state_h_in], outputs=rnn_out) + +stateful_rnn_out = REC_LAYER(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=True)(features_in, initial_state=state_h_in) +stateful_model = Model(inputs=[features_in, state_h_in], outputs=stateful_rnn_out) + +if USE_TOY_WEIGHTS: + if REC_LAYER == SimpleRNN: + toy_weights = [ np.asarray([[1.0]], dtype=np.float32), np.asarray([[-0.5]], dtype=np.float32)] + + elif REC_LAYER == GRU: + # for a GRU, the first are the non-recurrent kernels W, and the second are the recurrent kernels U (V) + toy_weights = [np.asarray([[ 1.0, -2.0, 3.0 ]], dtype=np.float32), np.asarray([[ -0.5 , 2.0, -1.1 ]], dtype=np.float32)] + + stateless_model.set_weights(toy_weights) + stateful_model.set_weights(toy_weights) + +# w = stateless_model.get_weights() +# print(w) + +stateless_model.save('temp_stateless.h5', include_optimizer=False) +stateful_model.save('temp_stateful.h5', include_optimizer=False) + +x_in = np.zeros(sequence_length) +x_in[0] = 1 +x_in = x_in.reshape( (1, sequence_length, feature_dim) ) +initial_state = np.asarray( [10]) +initial_state = initial_state.reshape((1,1)) + +def print_rnn_out(non_stateful_out, stateful_out): + fb = ['FWD::', 'BWD::'] + + print(f'non_stateful: {non_stateful_out}') + print(f'stateful: {stateful_out}') + print(f'delta: {stateful_out-non_stateful_out}') + +non_stateful_out = stateless_model.predict([x_in, initial_state]).reshape((sequence_length)) +stateful_out = stateful_model.predict([x_in, initial_state]).reshape((sequence_length)) +print_rnn_out(non_stateful_out, stateful_out) + +non_stateful_out = stateless_model.predict([x_in, initial_state]).reshape((sequence_length)) +stateful_out = stateful_model.predict([x_in, initial_state]).reshape((sequence_length)) +print_rnn_out(non_stateful_out, stateful_out) + +print('\n** RESETING STATES in STATEFUL MODEL **\n') +stateful_model.reset_states() +non_stateful_out = stateless_model.predict([x_in, initial_state]).reshape((sequence_length)) +stateful_out = stateful_model.predict([x_in, initial_state]).reshape((sequence_length)) +print_rnn_out(non_stateful_out, stateful_out) + +non_stateful_out = stateless_model.predict([x_in, initial_state]).reshape((sequence_length)) +stateful_out = stateful_model.predict([x_in, initial_state]).reshape((sequence_length)) +print_rnn_out(non_stateful_out, stateful_out) diff --git a/test/stateful_test/temp_stateful.h5 b/test/stateful_test/temp_stateful.h5 new file mode 100644 index 0000000000000000000000000000000000000000..7ba9b45f7aebe3c47e6922685a6f04f8c83dad8a GIT binary patch literal 13344 zcmeHNOK%)S5bm)_Sb~k#fD{J;3=2p>7-O$}uyTnJFh9OQ0>>DL|mE@a)iZDt8 zzqmX%CxDaXb4|ng0xEriZahs0H0C~BM0>B2(dc7n`D@XyheQKn1?9gWUKSN`%?a(uy6%LL8~8v!V;FKIMw8M(c`oZCpU9b!HY8?A zUPa1Vv0K+1zkQ$_8Mq_|p-A-_mU0vxt-$ZNUFZgRVqd*}JpoK4PlaecAD|2@1~8%C z{2n;2|2-+Y#owwD2gP;ctYR#mg@owp*dkDjYm+R!Nq)qJyZ%}dTa!Z7bA&yFIATNtk`jD)lz^*tM(d-T$&s+r^d|5CWvgu8DrhJB*OAV|$SUgwXDGopy%6y3zZrZ2?n}1|-*C2@=0;`Cy(@y9tc{TlGWJyI!KV z1mJ=6p1*vhfHfl^{Tln?&~|-$#X~IPFhMBG@D80g34JSazDXRvC6OX-?uqR<`#!pX zg)88?9kvlxeoYawbrLU~E%C#^!)IoR-JWCh?5~3`Lri=(MmgXXHM{s3jabiyFY zD=8HtOGBrdcy>sZqS`s7OS|Q)r9BdAH{P&Hdud8Y8Ig3f80^|LV_C19u&kpU25Uir zawa4`DcZ#zx3}gwJ#YthmAbSua=<@2v3$+w!r8pgkBVKd*yT724bPDV1FG@ zR3Ubri-AxzDD}w%U7_pNOjSRI&Fx$)hw{&voPwD4wVC_qV&R=2hkRUd)pp`bp)H z!Tx=~;+sko&vS#Ge|byMS+5;DOohUHx9D7RhxFwrsJ@=(o1y2crlKF0Pd=9};*jPS zyw@2O>V_*lm>w}N(0BD&G5x%lzDEbLd``U^A4WFcv)B3Yj;Z*ziCyO*0zy}%2<(-I zZkTy?j{C#PLmWTF{P4Krr~0x{9viOWU6r_!qZ_pXIDNa|!>F;Qo6JQ$35`XrXZa7D~~7SA7Uli%|Aq9135>ShK3 zh2sEaBOz|lej~$@YjT`dr*dTdAhs_62)BiporU@j_@Djj&mXbYMI)dQ&rv^dVLcVp_cgFl*v zC?ZFYK;ncr_Q{r9^F-zNdEI^US#h^RxXp#B%c%c3N%IH4U`SDi3&10U$83`5SuXi_#P&t-e$6*)7~MnsMD zm88B^yLrX&Tl?yfflGQ2ioRW=Qjfx;8Tf6t18mSI_O$EQV<1HORETEt0qVeJfDqcv z_krX3-{X=k?x;@e7uWUEO0akpQlh&fi$FfEP1@;o@+023>#rxVH7!(M6`e4VERLr+ z7JR^-<9Y>y7}^f4HzJlO4oASl$AxEaI-%v;T?ey?5HG=k&q8lNd}E`>Sg$8L$xOC# zulU|O9@FuSrgEIsPs!!Qn(BJL23TJdm%qHOM0R?TYOwI};sv=A3P821UJ4M^$?D|n z2XULzta?!pB6j>J2-{w;q4Y2gGJgPRtJ3uBD3Zuk&YG3aFfUgo%!*1yK-#Jxs>qW$ z3V9Oj2uREM>7U~5zkv55r(@NQn`@SWJ6f~XQQ`9RggG-|PB%bhE8di}3YcogGgX`( zt=nOQf^FN293X^tr{lCz1h$OsW@Q_M@-!g1{%Vl;Ez5_^NwXUu_`gj*FuSV-c1r*r z$nJ%UmvUG$0^rrii9_4LdA1^$Ce%y zI?W^u9Y40R5|P{K25t+tn?#Pa>e^|O;Gyrs+lK4Av1@zoci^4q*ICbB@PaUiFZpge z2)j8%BS*TOpLpIj`dHtLy>%{z@mkOcd^;af9k?a9)yXN)me-Ef?_=y&P8dX4Bc)-q z)6nT8o*mLo(d>-Tg>1QMAxA>(#+z0lmzIQl5lK&rVPCseEaJ)qix{miSPv4^GbQoq zqn+ot-F46Df;+IQROLP+2mGTQ%iD}ToK6cA6uUxec^T4#e0Bk>a97~p)^=mUWHJV1 zkKAY=izf85A|o;7oRVlH)J(WSwBbh5UvBP|pmZ7iljW4Rkt*St;kauk%`$jZ&p^<`7hkIFCCrSmxS^9!Eqj0<(b z)h|qsndj)6y(VU#7qj=MAj@^?-S{xFS{)3*M36atI`pGW4e@eKuOW9FMw3pd5dk~sN2EprTZhy0--r4Y}6u?-vL_vHZo z!Xn*Xq^B;@csMeUA0z;^2mPq=GJ>kp7k8OrCQ@* zMj)SWI7rWs|FG@Z7!_hr^rBB<%S~6r)Nj82aFhI&=jZ)6HB}E&2q+wfC>sfJgWfk% zBAFq_d3h#7))!*u9Ds0Bh*}N$KjDAv@4tS+UN?<^MnEH=5zq)|1db>IkMH~P^r8H| FybtKV{|Nv9 literal 0 HcmV?d00001 diff --git a/test/stateful_test/temp_stateless.json b/test/stateful_test/temp_stateless.json new file mode 100644 index 00000000..85273d22 --- /dev/null +++ b/test/stateful_test/temp_stateless.json @@ -0,0 +1,184 @@ +{ + "architecture": { + "backend": "tensorflow", + "class_name": "Model", + "config": { + "input_layers": [ + [ + "input_1", + 0, + 0 + ], + [ + "input_2", + 0, + 0 + ] + ], + "layers": [ + { + "class_name": "InputLayer", + "config": { + "batch_input_shape": [ + 1, + 3, + 1 + ], + "dtype": "float32", + "name": "input_1", + "sparse": false + }, + "inbound_nodes": [], + "name": "input_1" + }, + { + "class_name": "InputLayer", + "config": { + "batch_input_shape": [ + 1, + 1 + ], + "dtype": "float32", + "name": "input_2", + "sparse": false + }, + "inbound_nodes": [], + "name": "input_2" + }, + { + "class_name": "GRU", + "config": { + "activation": "linear", + "activity_regularizer": null, + "bias_constraint": null, + "bias_initializer": { + "class_name": "Zeros", + "config": {} + }, + "bias_regularizer": null, + "dropout": 0.0, + "dtype": "float32", + "go_backwards": false, + "implementation": 2, + "kernel_constraint": null, + "kernel_initializer": { + "class_name": "GlorotUniform", + "config": { + "seed": null + } + }, + "kernel_regularizer": null, + "name": "gru", + "recurrent_activation": "sigmoid", + "recurrent_constraint": null, + "recurrent_dropout": 0.0, + "recurrent_initializer": { + "class_name": "Orthogonal", + "config": { + "gain": 1.0, + "seed": null + } + }, + "recurrent_regularizer": null, + "reset_after": true, + "return_sequences": true, + "return_state": false, + "stateful": false, + "time_major": false, + "trainable": true, + "units": 1, + "unroll": false, + "use_bias": false + }, + "inbound_nodes": [ + [ + [ + "input_1", + 0, + 0, + {} + ], + [ + "input_2", + 0, + 0, + {} + ] + ] + ], + "name": "gru" + } + ], + "name": "model", + "output_layers": [ + [ + "gru", + 0, + 0 + ] + ] + }, + "keras_version": "2.2.4-tf" + }, + "hash": "9f36a6b98012df2ef10ea88aeb0589da9c1050a9c5875a64432e13e336e72536", + "image_data_format": "channels_last", + "input_shapes": [ + [ + 3, + 1 + ], + [ + 1 + ] + ], + "output_shapes": [ + [ + 3, + 1 + ] + ], + "tests": [ + { + "inputs": [ + { + "shape": [ + 3, + 1 + ], + "values": [ + "QYkivxu6Jb9bk3q/" + ] + }, + { + "shape": [ + 1 + ], + "values": [ + "o2GAvw==" + ] + } + ], + "outputs": [ + { + "shape": [ + 3, + 1 + ], + "values": [ + "CIylv9xzu7/NtgLA" + ] + } + ] + } + ], + "trainable_params": { + "gru": { + "recurrent_weights": [ + "AAAAvwAAAEDNzIy/" + ], + "weights": [ + "AACAPwAAAMAAAEBA" + ] + } + } +} From 025a8db020eb4c848355ad086510ce84ff241236 Mon Sep 17 00:00:00 2001 From: Keith Chugg Date: Wed, 8 Apr 2020 03:05:00 +0000 Subject: [PATCH 3/7] WIP: GRU/LSTM work. Issues with bidi --- include/fdeep/layers/bidirectional_layer.hpp | 72 ++++++++++--------- include/fdeep/layers/gru_layer.hpp | 26 +++++-- include/fdeep/layers/lstm_layer.hpp | 33 ++++++--- .../stateful_recurrent_tests.cpp | 2 +- test/stateful_test/temp_rnn_results.txt | 23 +++++- 5 files changed, 102 insertions(+), 54 deletions(-) diff --git a/include/fdeep/layers/bidirectional_layer.hpp b/include/fdeep/layers/bidirectional_layer.hpp index 31d46a57..53758e91 100644 --- a/include/fdeep/layers/bidirectional_layer.hpp +++ b/include/fdeep/layers/bidirectional_layer.hpp @@ -56,7 +56,9 @@ class bidirectional_layer : public layer forward_state_h_(stateful ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()), forward_state_c_(stateful && wrapped_layer_type_has_state_c(wrapped_layer_type) ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()), backward_state_h_(stateful ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()), - backward_state_c_(stateful && wrapped_layer_type_has_state_c(wrapped_layer_type) ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()) + backward_state_c_(stateful && wrapped_layer_type_has_state_c(wrapped_layer_type) ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()), + use_avail_input_state_for_stateful_(true) + { } @@ -67,6 +69,7 @@ class bidirectional_layer : public layer forward_state_c_ = tensor(tensor_shape(n_units_), static_cast(0)); backward_state_h_ = tensor(tensor_shape(n_units_), static_cast(0)); backward_state_c_ = tensor(tensor_shape(n_units_), static_cast(0)); + use_avail_input_state_for_stateful_ = true; } } @@ -110,29 +113,26 @@ class bidirectional_layer : public layer assertion(inputs.size() == 1 || inputs.size() == 5, "Invalid number of input tensors."); - tensor forward_state_h = inputs.size() == 5 - ? inputs[1] - : is_stateful() - ? forward_state_h_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); - - tensor forward_state_c = inputs.size() == 5 - ? inputs[2] - : is_stateful() - ? forward_state_c_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); - - tensor backward_state_h = inputs.size() == 5 - ? inputs[3] - : is_stateful() - ? backward_state_h_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); - - tensor backward_state_c = inputs.size() == 5 - ? inputs[4] - : is_stateful() - ? backward_state_c_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); + bool initial_state_provided = inputs.size() == 5; + bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; + bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state; + // bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state; + + tensor forward_state_h = use_input_initial_state ? inputs[1] : + use_last_state_for_initial_state ? forward_state_h_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state + + tensor forward_state_c = use_input_initial_state ? inputs[2] : + use_last_state_for_initial_state ? forward_state_c_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state + + tensor backward_state_h = use_input_initial_state ? inputs[3] : + use_last_state_for_initial_state ? backward_state_h_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state + + tensor backward_state_c = use_input_initial_state ? inputs[4] : + use_last_state_for_initial_state ? backward_state_c_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state result_forward = lstm_impl(input, forward_state_h, forward_state_c, n_units_, use_bias_, return_sequences_, stateful_, @@ -147,6 +147,7 @@ class bidirectional_layer : public layer forward_state_c_ = forward_state_c; backward_state_h_ = backward_state_h; backward_state_c_ = backward_state_c; + use_avail_input_state_for_stateful_ = false; } } else if (wrapped_layer_type_ == "GRU" || wrapped_layer_type_ == "CuDNNGRU") @@ -154,17 +155,18 @@ class bidirectional_layer : public layer assertion(inputs.size() == 1 || inputs.size() == 3, "Invalid number of input tensors."); - tensor forward_state_h = inputs.size() == 3 - ? inputs[1] - : is_stateful() - ? forward_state_h_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); + bool initial_state_provided = inputs.size() == 3; + bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; + bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state; + // bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state; + + tensor forward_state_h = use_input_initial_state ? inputs[1] : + use_last_state_for_initial_state ? forward_state_h_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state - tensor backward_state_h = inputs.size() == 3 - ? inputs[2] - : is_stateful() - ? backward_state_h_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); + tensor backward_state_h = use_input_initial_state ? inputs[2] : + use_last_state_for_initial_state ? backward_state_h_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state result_forward = gru_impl(input, forward_state_h, n_units_, use_bias_, reset_after_, return_sequences_, false, forward_weights_, forward_recurrent_weights_, @@ -175,6 +177,7 @@ class bidirectional_layer : public layer if (is_stateful()) { forward_state_h_ = forward_state_h; backward_state_h_ = backward_state_h; + use_avail_input_state_for_stateful_ = false; } } else @@ -223,6 +226,7 @@ class bidirectional_layer : public layer mutable fplus::maybe forward_state_c_; mutable fplus::maybe backward_state_h_; mutable fplus::maybe backward_state_c_; + mutable bool use_avail_input_state_for_stateful_; }; } // namespace internal diff --git a/include/fdeep/layers/gru_layer.hpp b/include/fdeep/layers/gru_layer.hpp index e561f772..3e5125db 100644 --- a/include/fdeep/layers/gru_layer.hpp +++ b/include/fdeep/layers/gru_layer.hpp @@ -44,7 +44,8 @@ class gru_layer : public layer weights_(weights), recurrent_weights_(recurrent_weights), bias_(bias), - state_h_(stateful ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()) + state_h_(stateful ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()), + use_avail_input_state_for_stateful_(true) { } @@ -53,6 +54,7 @@ class gru_layer : public layer { if (is_stateful()) { state_h_ = tensor(tensor_shape(n_units_), static_cast(0)); + use_avail_input_state_for_stateful_ = true; } } @@ -77,17 +79,26 @@ class gru_layer : public layer assertion(inputs.size() == 1 || inputs.size() == 2, "Invalid number of input tensors."); - tensor state_h = inputs.size() == 2 - ? inputs[1] - : is_stateful() - ? state_h_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); - + // RNN behaivor since TF 2.1: + // If an *initial state input is provided*, this is used always for non-stateful models + // but only on reset for stateful models (including the very first call) + // If *no input state is provided*, then initial state is 0 for non-stateful + // and, for stateful, it carries the state from previous call, unless state-reset, in which case it set to 0 + bool initial_state_provided = inputs.size() == 2; + bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; + bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state; + // bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state; + + tensor state_h = use_input_initial_state ? inputs[1] : + use_last_state_for_initial_state ? state_h_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state + const auto result = gru_impl(input, state_h, n_units_, use_bias_, reset_after_, return_sequences_, return_state_, weights_, recurrent_weights_, bias_, activation_, recurrent_activation_); if (is_stateful()) { state_h_ = state_h; + use_avail_input_state_for_stateful_ = false; } return result; } @@ -104,6 +115,7 @@ class gru_layer : public layer const float_vec recurrent_weights_; const float_vec bias_; mutable fplus::maybe state_h_; + mutable bool use_avail_input_state_for_stateful_; }; } // namespace internal diff --git a/include/fdeep/layers/lstm_layer.hpp b/include/fdeep/layers/lstm_layer.hpp index 9dade051..acd464d9 100644 --- a/include/fdeep/layers/lstm_layer.hpp +++ b/include/fdeep/layers/lstm_layer.hpp @@ -43,7 +43,9 @@ class lstm_layer : public layer recurrent_weights_(recurrent_weights), bias_(bias), state_h_(stateful ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()), - state_c_(stateful ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()) + state_c_(stateful ? tensor(tensor_shape(n_units), static_cast(0)) : fplus::nothing()), + use_avail_input_state_for_stateful_(true) + { } @@ -52,6 +54,7 @@ class lstm_layer : public layer if (is_stateful()) { state_h_ = tensor(tensor_shape(n_units_), static_cast(0)); state_c_ = tensor(tensor_shape(n_units_), static_cast(0)); + use_avail_input_state_for_stateful_ = true; } } @@ -74,17 +77,23 @@ class lstm_layer : public layer assertion(inputs.size() == 1 || inputs.size() == 3, "Invalid number of input tensors."); - tensor state_h = inputs.size() == 3 - ? inputs[1] - : is_stateful() - ? state_h_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); + // RNN behaivor since TF 2.1: + // If an *initial state input is provided*, this is used always for non-stateful models + // but only on reset for stateful models (including the very first call) + // If *no input state is provided*, then initial state is 0 for non-stateful + // and, for stateful, it carries the state from previous call, unless state-reset, in which case it set to 0 + bool initial_state_provided = inputs.size() == 3; + bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; + bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state; + // bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state; + + tensor state_h = use_input_initial_state ? inputs[1] : + use_last_state_for_initial_state ? state_h_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state - tensor state_c = inputs.size() == 3 - ? inputs[2] - : is_stateful() - ? state_c_.unsafe_get_just() - : tensor(tensor_shape(n_units_), static_cast(0)); + tensor state_c = use_input_initial_state ? inputs[2] : + use_last_state_for_initial_state ? state_c_.unsafe_get_just() : + tensor(tensor_shape(n_units_), static_cast(0)); // use_zero_initial_state const auto result = lstm_impl(input, state_h, state_c, n_units_, use_bias_, return_sequences_, return_state_, weights_, @@ -92,6 +101,7 @@ class lstm_layer : public layer if (is_stateful()) { state_h_ = state_h; state_c_ = state_c; + use_avail_input_state_for_stateful_ = false; } return result; } @@ -108,6 +118,7 @@ class lstm_layer : public layer const float_vec bias_; mutable fplus::maybe state_h_; mutable fplus::maybe state_c_; + mutable bool use_avail_input_state_for_stateful_; }; } // namespace internal diff --git a/test/stateful_test/stateful_recurrent_tests.cpp b/test/stateful_test/stateful_recurrent_tests.cpp index 57ad200e..f89e60f0 100644 --- a/test/stateful_test/stateful_recurrent_tests.cpp +++ b/test/stateful_test/stateful_recurrent_tests.cpp @@ -158,7 +158,7 @@ int main() vec_append(all_results, *result[0].as_vector()); // ************************* BIDIRECTIONAL TESTS ************************* // - #define TF_BIDI_BUG_FIXED false + #define TF_BIDI_BUG_FIXED true // *********** TEST 9: "bidi-GRU_nonstateful_no_init_state.json" *********** model = load_model("./models/bidi-GRU_nonstateful_no_init_state.json"); /// state_reset = true diff --git a/test/stateful_test/temp_rnn_results.txt b/test/stateful_test/temp_rnn_results.txt index a2d01f79..5f803302 100644 --- a/test/stateful_test/temp_rnn_results.txt +++ b/test/stateful_test/temp_rnn_results.txt @@ -55,4 +55,25 @@ Stateful Non-Stateful [[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] \ No newline at end of file +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] + +After Mods: +-------------------------------------------- +starting A +Non-Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Non-Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Stateful +[[[[[[[-7.2471], [-7.0587], [-6.8576]]]]]]] +** RESETING STATES in STATEFUL MODEL ** +Non-Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Non-Stateful +[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] +Stateful +[[[[[[[-7.2471], [-7.0587], [-6.8576]]]]]]] \ No newline at end of file From 4336101f99c923aea057236a4467ae94a64c3877 Mon Sep 17 00:00:00 2001 From: Keith Chugg Date: Thu, 9 Apr 2020 05:12:17 +0000 Subject: [PATCH 4/7] WIP: stateful stateful RNNs... --- include/fdeep/layers/bidirectional_layer.hpp | 4 +- .../stateful_recurrent_tests.cpp | 54 ++++++---- .../stateful_test/stateful_recurrent_tests.py | 16 ++- test/stateful_test/temp_bidi_no_state_in.py | 48 +++++++++ test/stateful_test/temp_bidi_state_in.py | 81 ++++++++++++++ test/stateful_test/temp_old_bidi_full.py | 100 ++++++++++++++++++ 6 files changed, 278 insertions(+), 25 deletions(-) create mode 100644 test/stateful_test/temp_bidi_no_state_in.py create mode 100644 test/stateful_test/temp_bidi_state_in.py create mode 100644 test/stateful_test/temp_old_bidi_full.py diff --git a/include/fdeep/layers/bidirectional_layer.hpp b/include/fdeep/layers/bidirectional_layer.hpp index 53758e91..23286d1d 100644 --- a/include/fdeep/layers/bidirectional_layer.hpp +++ b/include/fdeep/layers/bidirectional_layer.hpp @@ -114,7 +114,7 @@ class bidirectional_layer : public layer "Invalid number of input tensors."); bool initial_state_provided = inputs.size() == 5; - bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; + bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state; // bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state; @@ -156,7 +156,7 @@ class bidirectional_layer : public layer "Invalid number of input tensors."); bool initial_state_provided = inputs.size() == 3; - bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; + bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state; // bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state; diff --git a/test/stateful_test/stateful_recurrent_tests.cpp b/test/stateful_test/stateful_recurrent_tests.cpp index f89e60f0..05e321b0 100644 --- a/test/stateful_test/stateful_recurrent_tests.cpp +++ b/test/stateful_test/stateful_recurrent_tests.cpp @@ -16,13 +16,23 @@ int main() const std::vector x_inf_0 = {2.1, -1.2, 3.14, 1.2}; const std::vector x_inf_1 = {1, 3, -2, 10}; - const std::vector state_0 = {1.1, -2.1}; - const std::vector state_1 = {2.7, 3.1}; - const std::vector state_2 = {-2.5, 3.0}; - const std::vector state_3 = {-2.0, -10.0}; + const std::vector state_0 = {40.1, -25.1}; + const std::vector state_1 = {34.7, 56.1}; + const std::vector state_2 = {-62.5, 12.0}; + const std::vector state_3 = {-33.0, -100.0}; + + + + // const std::vector state_0 = {1.1, -2.1}; + // const std::vector state_1 = {2.7, 3.1}; + // const std::vector state_2 = {-2.5, 3.0}; + // const std::vector state_3 = {-2.0, -10.0}; std::vector all_results = {}; std::vector one_result = {}; +// [40.1, -25.1, 34.7, 56.1, -62.5, 12.0, -33.0, -100.0] +// [1.1, -2.1, 2.7, 3.1, -2.5, 3.0, -2.0, -10.0] + const shared_float_vec xt0(fplus::make_shared_ref(x_inf_0)); const shared_float_vec xt1(fplus::make_shared_ref(x_inf_1)); const shared_float_vec st0(fplus::make_shared_ref(state_0)); @@ -158,16 +168,16 @@ int main() vec_append(all_results, *result[0].as_vector()); // ************************* BIDIRECTIONAL TESTS ************************* // - #define TF_BIDI_BUG_FIXED true + #define TF_BIDI_STATE_RESET_WORKS false // *********** TEST 9: "bidi-GRU_nonstateful_no_init_state.json" *********** model = load_model("./models/bidi-GRU_nonstateful_no_init_state.json"); /// state_reset = true result = model.predict({test_in_0}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); result = model.predict({test_in_1}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); /// state_reset = false result = model.predict({test_in_0}); vec_append(all_results, *result[0].as_vector()); @@ -179,10 +189,10 @@ int main() /// state_reset = true result = model.predict({test_in_0, test_state_0, test_state_1}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); result = model.predict({test_in_1, test_state_0, test_state_1}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); /// state_reset = false result = model.predict({test_in_0, test_state_0, test_state_1}); vec_append(all_results, *result[0].as_vector()); @@ -194,10 +204,10 @@ int main() /// state_reset = true result = model.predict_stateful({test_in_0}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); result = model.predict_stateful({test_in_1}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); /// state_reset = false result = model.predict_stateful({test_in_0}); vec_append(all_results, *result[0].as_vector()); @@ -209,10 +219,10 @@ int main() /// state_reset = true result = model.predict_stateful({test_in_0, test_state_0, test_state_1}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); result = model.predict_stateful({test_in_1, test_state_0, test_state_1}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); /// state_reset = false result = model.predict_stateful({test_in_0, test_state_0, test_state_1}); vec_append(all_results, *result[0].as_vector()); @@ -224,10 +234,10 @@ int main() /// state_reset = true result = model.predict({test_in_0}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); result = model.predict({test_in_1}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); /// state_reset = false result = model.predict({test_in_0}); vec_append(all_results, *result[0].as_vector()); @@ -239,10 +249,10 @@ int main() /// state_reset = true result = model.predict({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); result = model.predict({test_in_1, test_state_0, test_state_1, test_state_2, test_state_3}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); /// state_reset = false result = model.predict({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); vec_append(all_results, *result[0].as_vector()); @@ -254,10 +264,10 @@ int main() /// state_reset = true result = model.predict_stateful({test_in_0}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); result = model.predict_stateful({test_in_1}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); /// state_reset = false result = model.predict_stateful({test_in_0}); vec_append(all_results, *result[0].as_vector()); @@ -269,17 +279,17 @@ int main() /// state_reset = true result = model.predict_stateful({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); result = model.predict_stateful({test_in_1, test_state_0, test_state_1, test_state_2, test_state_3}); vec_append(all_results, *result[0].as_vector()); - if(TF_BIDI_BUG_FIXED) model.reset_states(); + if(TF_BIDI_STATE_RESET_WORKS) model.reset_states(); /// state_reset = false result = model.predict_stateful({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); vec_append(all_results, *result[0].as_vector()); result = model.predict_stateful({test_in_1, test_state_0, test_state_1, test_state_2, test_state_3}); vec_append(all_results, *result[0].as_vector()); - #undef TF_BIDI_BUG_FIXED + #undef TF_BIDI_STATE_RESET_WORKS if(verbose){ std::cout << "\n\nOUTPUT ***" << std::endl; diff --git a/test/stateful_test/stateful_recurrent_tests.py b/test/stateful_test/stateful_recurrent_tests.py index 5d5defb0..f0828a1f 100644 --- a/test/stateful_test/stateful_recurrent_tests.py +++ b/test/stateful_test/stateful_recurrent_tests.py @@ -1,3 +1,8 @@ + +import os +os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' +os.environ['CUDA_VISIBLE_DEVICES']='' + import errno import os import sys @@ -176,6 +181,10 @@ def main(): # Memory growth must be set before GPUs have been initialized print(e) + #### DEBUG.... + # rm old ./models if there and create new one. + os.system('rm -rf ./models') + os.makedirs('./models') # generate toy data train_seq_length = 4 feature_dim = 1 @@ -194,7 +203,8 @@ def main(): x_inf = np.asarray([2.1, -1.2, 3.14, 1.2, 1, 3, -2, 10], dtype=np.float32) # simple x_inf = x_inf.reshape((2, train_seq_length, 1)) - initial_states = np.asarray([1.1, -2.1, 2.7, 3.1, -2.5, 3.0, -2.0, -10.0], dtype=np.float32) + initial_states = np.asarray([40.1, -25.1, 34.7, 56.1, -62.5, 12.0, -33.0, -100.0], dtype=np.float32) + # initial_states = np.asarray([1.1, -2.1, 2.7, 3.1, -2.5, 3.0, -2.0, -10.0], dtype=np.float32) initial_states = initial_states.reshape((4, 1, 2)) model_file_names = [] @@ -253,6 +263,7 @@ def main(): print('********* FAILED !!!!!!!!!!!!\n\n') print('Keras: ', all_results[i], '\n') print('Frugally-deep: ', frugally_deep_results[i], '\n') + print('Difference: ', all_results[i] - frugally_deep_results[i], '\n') all_tests_passed = False if not all_tests_passed: @@ -260,6 +271,9 @@ def main(): sys.exit(errno.EIO) print('\n\nPassed all stateful tests') + ## debug + all_results.to_file('models/keras_results.npy') + if __name__ == "__main__": main() diff --git a/test/stateful_test/temp_bidi_no_state_in.py b/test/stateful_test/temp_bidi_no_state_in.py new file mode 100644 index 00000000..3ec648d5 --- /dev/null +++ b/test/stateful_test/temp_bidi_no_state_in.py @@ -0,0 +1,48 @@ +import os +os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' +os.environ['CUDA_VISIBLE_DEVICES']='' + +import numpy as np +from tensorflow.keras.layers import Input, Dense, SimpleRNN, GRU, LSTM, Bidirectional +from tensorflow.keras.models import Model + +REC = LSTM + +sequence_length = 3 +feature_dim = 1 +features_in = Input(batch_shape=(1, sequence_length, feature_dim)) + +rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=False))(features_in) +stateless_model = Model(inputs=[features_in], outputs=[rnn_out]) + +stateful_rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=True))(features_in) +stateful_model = Model(inputs=features_in, outputs=stateful_rnn_out) + +stateful_model.set_weights( stateless_model.get_weights() ) + +x_in = np.random.normal(0,10,sequence_length) +x_in = x_in.reshape( (1, sequence_length, feature_dim) ) + +def print_bidi_out(non_stateful_out, stateful_out): + fb = ['FWD::', 'BWD::'] + + for i in range(2): + print(fb[i]) + print(f'non_stateful: {non_stateful_out.T[i]}') + print(f'stateful: {stateful_out.T[i]}') + print(f'delta: {stateful_out.T[i]-non_stateful_out.T[i]}') + + +non_stateful_out = stateless_model.predict(x_in).reshape((sequence_length,2)) +stateful_out = stateful_model.predict(x_in).reshape((sequence_length,2)) +print_bidi_out(non_stateful_out, stateful_out) + +non_stateful_out = stateless_model.predict(x_in).reshape((sequence_length,2)) +stateful_out = stateful_model.predict(x_in).reshape((sequence_length,2)) +print_bidi_out(non_stateful_out, stateful_out) + +print('\n** RESETING STATES in STATEFUL MODEL **\n') +stateful_model.reset_states() +non_stateful_out = stateless_model.predict(x_in).reshape((sequence_length,2)) +stateful_out = stateful_model.predict(x_in).reshape((sequence_length,2)) +print_bidi_out(non_stateful_out, stateful_out) diff --git a/test/stateful_test/temp_bidi_state_in.py b/test/stateful_test/temp_bidi_state_in.py new file mode 100644 index 00000000..ec0b2a76 --- /dev/null +++ b/test/stateful_test/temp_bidi_state_in.py @@ -0,0 +1,81 @@ +import os +os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' +os.environ['CUDA_VISIBLE_DEVICES']='' + +import numpy as np +from tensorflow.keras.layers import Input, Dense, SimpleRNN, GRU, LSTM, Bidirectional +from tensorflow.keras.models import Model + +REC = SimpleRNN + +sequence_length = 3 +feature_dim = 1 +features_in = Input(batch_shape=(1, sequence_length, feature_dim)) +state_h_fwd_in = Input(batch_shape=(1, 1)) +state_h_bwd_in = Input(batch_shape=(1, 1)) +state_c_fwd_in = Input(batch_shape=(1, 1)) +state_c_bwd_in = Input(batch_shape=(1, 1)) + +four_state_shape = [state_h_fwd_in, state_c_fwd_in, state_h_bwd_in, state_c_bwd_in] +two_state_shape = [state_h_fwd_in, state_h_bwd_in] + +if REC == LSTM: + rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=False))(features_in, initial_state=four_state_shape) + stateful_rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=True))(features_in, initial_state=four_state_shape) + rnn_inputs = [features_in, state_h_fwd_in, state_c_fwd_in, state_h_bwd_in, state_c_bwd_in] +else: + rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=False))(features_in, initial_state=two_state_shape) + stateful_rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=True))(features_in, initial_state=two_state_shape) + rnn_inputs = [features_in, state_h_fwd_in, state_h_bwd_in] + +stateless_model = Model(inputs=rnn_inputs, outputs=rnn_out) +stateful_model = Model(inputs=rnn_inputs, outputs=stateful_rnn_out) + + +toy_weights = [np.asarray([[ 1.0]], dtype=np.float32), np.asarray([[0.5 ]], dtype=np.float32), np.asarray([[ 0.0 ]], dtype=np.float32), np.asarray([[ 0.0 ]], dtype=np.float32)] +stateless_model.set_weights(toy_weights) +stateful_model.set_weights(toy_weights) + +# stateful_model.set_weights( stateless_model.get_weights() ) + +x_in = np.random.normal(0,10,sequence_length) +x_in = np.asarray([1,0,0]) +x_in = x_in.reshape( (1, sequence_length, feature_dim) ) + +fwd_initial_h = np.asarray(100.0).reshape(1,1) +fwd_initial_h = np.asarray(12.75).reshape(1,1) + + +bwd_initial_h = np.asarray(-100.0).reshape(1,1) +fwd_initial_c = np.asarray(50.0).reshape(1,1) +bwd_initial_c = np.asarray(-50.0).reshape(1,1) + +if REC == LSTM: + rnn_input = [x_in, fwd_initial_h, fwd_initial_c, bwd_initial_h, bwd_initial_c] +else: + rnn_input = [x_in, fwd_initial_h, bwd_initial_h] + + +def print_bidi_out(non_stateful_out, stateful_out): + fb = ['FWD::', 'BWD::'] + + for i in range(2): + print(fb[i]) + print(f'non_stateful: {non_stateful_out.T[i]}') + print(f'stateful: {stateful_out.T[i]}') + print(f'delta: {stateful_out.T[i]-non_stateful_out.T[i]}') + + +non_stateful_out = stateless_model.predict(rnn_input).reshape((sequence_length,2)) +stateful_out = stateful_model.predict(rnn_input).reshape((sequence_length,2)) +print_bidi_out(non_stateful_out, stateful_out) + +non_stateful_out = stateless_model.predict(rnn_input).reshape((sequence_length,2)) +stateful_out = stateful_model.predict(rnn_input).reshape((sequence_length,2)) +print_bidi_out(non_stateful_out, stateful_out) + +print('\n** RESETING STATES in STATEFUL MODEL **\n') +stateful_model.reset_states() +non_stateful_out = stateless_model.predict(rnn_input).reshape((sequence_length,2)) +stateful_out = stateful_model.predict(rnn_input).reshape((sequence_length,2)) +print_bidi_out(non_stateful_out, stateful_out) diff --git a/test/stateful_test/temp_old_bidi_full.py b/test/stateful_test/temp_old_bidi_full.py new file mode 100644 index 00000000..5419b8a4 --- /dev/null +++ b/test/stateful_test/temp_old_bidi_full.py @@ -0,0 +1,100 @@ +import h5py +import numpy as np +from tensorflow.keras.layers import Input, Dense, LSTM, Bidirectional +from tensorflow.keras import Model + +#### generate toy data +stream_seq_length = 4 +seq_ratio = 2 +train_seq_length = stream_seq_length * seq_ratio +feature_dim = 2 +num_seqs = 3 + +x = np.random.randint(0, high=2, size = (num_seqs * train_seq_length, feature_dim) ) +x = np.sign( x - 0.5 ) +y = np.sum( ( x == np.roll(x, 1, axis = 0) ), axis = 1 ) +### y[n] = number of agreements between x[n], x[n-1] +x = x.reshape( (num_seqs, train_seq_length, feature_dim) ) +y = y.reshape( (num_seqs, train_seq_length, 1) ) + + +###### Define/Build/Train Training Model +lstm_output_dim = 4 + +training_in_shape = x.shape[1:] +training_in = Input(shape=training_in_shape) +# training_in = Input(batch_shape=(None,train_seq_length,feature_dim)) this works too +foo = Bidirectional( LSTM(lstm_output_dim, return_sequences=True, stateful=False) ) (training_in) +training_pred = Dense(1)(foo) +training_model = Model(inputs=training_in, outputs=training_pred) +training_model.compile(loss='mean_squared_error', optimizer='adam') +training_model.summary() +training_model.fit(x, y, batch_size=2, epochs=10) + +training_model.save_weights('weights.hd5', overwrite=True) + +##### define the streaming-infernece model +streaming_in = Input(batch_shape=(1,stream_seq_length,feature_dim)) ## stateful ==> needs batch_shape specified +initial_state_h = Input(batch_shape=(1,lstm_output_dim)) +initial_state_c = Input(batch_shape=(1,lstm_output_dim)) + +fwd_initial_state_h = Input(batch_shape=(1,lstm_output_dim)) +fwd_initial_state_c = Input(batch_shape=(1,lstm_output_dim)) +bwd_initial_state_h = Input(batch_shape=(1,lstm_output_dim)) +bwd_initial_state_c = Input(batch_shape=(1,lstm_output_dim)) + + +lstm_streaming_stateful = Bidirectional( LSTM(lstm_output_dim, return_sequences=True, stateful=True ) ) (streaming_in) +lstm_streaming_initial_stateful = Bidirectional( LSTM(lstm_output_dim, return_sequences=True, stateful=True) ) (streaming_in, initial_state=[fwd_initial_state_h, fwd_initial_state_c, bwd_initial_state_h, bwd_initial_state_c]) +lstm_streaming_initial_nonstateful = Bidirectional( LSTM(lstm_output_dim, return_sequences=True, stateful=False) ) (streaming_in, initial_state=[fwd_initial_state_h, fwd_initial_state_c, bwd_initial_state_h, bwd_initial_state_c]) + + +streaming_stateful_model = Model( inputs=streaming_in, outputs=Dense(1)(lstm_streaming_stateful) ) +streaming_initial_stateful_model = Model( inputs=[streaming_in,fwd_initial_state_h, fwd_initial_state_c, bwd_initial_state_h, bwd_initial_state_c], outputs=Dense(1)(lstm_streaming_initial_stateful) ) +streaming_initial_nonstateful_model = Model( inputs=[streaming_in, fwd_initial_state_h, fwd_initial_state_c, bwd_initial_state_h, bwd_initial_state_c], outputs=Dense(1)(lstm_streaming_initial_nonstateful) ) + +streaming_stateful_model.compile(loss='mean_squared_error', optimizer='adam') +streaming_initial_stateful_model.compile(loss='mean_squared_error', optimizer='adam') +streaming_initial_nonstateful_model.compile(loss='mean_squared_error', optimizer='adam') + +streaming_stateful_model.load_weights('weights.hd5') +streaming_initial_stateful_model.load_weights('weights.hd5') +streaming_initial_nonstateful_model.load_weights('weights.hd5') + + +##### demo the behaivor + +fwd_initial_c = np.random.normal(0,1, (1,lstm_output_dim)) +fwd_initial_h = np.random.normal(0,1, (1,lstm_output_dim)) +bwd_initial_c = np.random.normal(0,1, (1,lstm_output_dim)) +bwd_initial_h = np.random.normal(0,1, (1,lstm_output_dim)) +# initial_c = np.zeros((1,lstm_output_dim)) +# initial_h = np.zeros((1,lstm_output_dim)) + +state_reset = True + +print('\n\n******the streaming-inference model can replicate the sequence-based trained model:\n') +for s in range(num_seqs): + if state_reset: + print(f'\n\nRunning Sequence {s} with STATE RESET:\n') + else: + print(f'\n\nRunning Sequence {s} with NO STATE RESET:\n') + in_seq = x[s].reshape( (1, train_seq_length, feature_dim) ) + seq_pred = training_model.predict(in_seq) + seq_pred = seq_pred.reshape(train_seq_length) + for k in range(seq_ratio): + in_feature_vector = x[s][ k * stream_seq_length : (k+1) * stream_seq_length ].reshape(1,stream_seq_length,feature_dim) + stream_stateful_pred = streaming_stateful_model.predict(in_feature_vector).reshape(stream_seq_length) + stream_stateful_init_pred = streaming_initial_stateful_model.predict([in_feature_vector, fwd_initial_c, fwd_initial_h, bwd_initial_c, bwd_initial_h]).reshape(stream_seq_length) + stream_nonstateful_init_pred = streaming_initial_nonstateful_model.predict([in_feature_vector, fwd_initial_c, fwd_initial_h, bwd_initial_c, bwd_initial_h]).reshape(stream_seq_length) + seq_pred_chunk = seq_pred[k * stream_seq_length : (k+1) * stream_seq_length] + for n in range(stream_seq_length): + msg = f'Seq-model Prediction[{k}, {n}]: { seq_pred_chunk[n] : 3.2f}\n' + msg += f'no-init-stateful, diff: {stream_stateful_pred[n] : 3.2f}, {seq_pred_chunk[n] - stream_stateful_pred[n] : 3.2f}\n' + msg += f'init-stateful, diff: {stream_stateful_init_pred[n] : 3.2f}, {seq_pred_chunk[n] - stream_stateful_init_pred[n] : 3.2f}\n' + msg += f'init-not-stateful, diff: {stream_nonstateful_init_pred[n] : 3.2f}, {seq_pred_chunk[n] - stream_nonstateful_init_pred[n] : 3.2f}\n\n' + print(msg) + if state_reset: + streaming_stateful_model.reset_states() + # streaming_initial_stateful_model.reset_states() <<----- this causes an error. + streaming_initial_nonstateful_model.reset_states() From 691ccdf58e31d09b45af2616cc741d4e9fadfb7e Mon Sep 17 00:00:00 2001 From: Keith Chugg Date: Thu, 9 Apr 2020 19:16:03 +0000 Subject: [PATCH 5/7] TF 2.1 compatibility -- stateful_recurrent_tests all pass --- include/fdeep/layers/bidirectional_layer.hpp | 3 + .../stateful_recurrent_tests.cpp | 2 +- .../stateful_test/stateful_recurrent_tests.py | 16 +- test/stateful_test/temp_old_bidi_full.py | 100 ---------- test/stateful_test/temp_rnn_results.txt | 79 -------- test/stateful_test/temp_stateful.h5 | Bin 13344 -> 0 bytes test/stateful_test/temp_stateful.json | 184 ------------------ test/stateful_test/temp_stateless.h5 | Bin 13344 -> 0 bytes test/stateful_test/temp_stateless.json | 184 ------------------ .../temp_bidi_no_state_in.py | 0 .../temp_bidi_state_in.cpp | 72 +++++++ .../temp_bidi_state_in.py | 41 ++-- .../temp_rnn_test.cpp | 0 .../temp_rnn_test.py | 0 14 files changed, 105 insertions(+), 576 deletions(-) delete mode 100644 test/stateful_test/temp_old_bidi_full.py delete mode 100644 test/stateful_test/temp_rnn_results.txt delete mode 100644 test/stateful_test/temp_stateful.h5 delete mode 100644 test/stateful_test/temp_stateful.json delete mode 100644 test/stateful_test/temp_stateless.h5 delete mode 100644 test/stateful_test/temp_stateless.json rename test/stateful_test/{ => tf_behaivor_scripts}/temp_bidi_no_state_in.py (100%) create mode 100644 test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.cpp rename test/stateful_test/{ => tf_behaivor_scripts}/temp_bidi_state_in.py (52%) rename test/stateful_test/{ => tf_behaivor_scripts}/temp_rnn_test.cpp (100%) rename test/stateful_test/{ => tf_behaivor_scripts}/temp_rnn_test.py (100%) diff --git a/include/fdeep/layers/bidirectional_layer.hpp b/include/fdeep/layers/bidirectional_layer.hpp index 23286d1d..02aed313 100644 --- a/include/fdeep/layers/bidirectional_layer.hpp +++ b/include/fdeep/layers/bidirectional_layer.hpp @@ -64,6 +64,9 @@ class bidirectional_layer : public layer void reset_states() override { + // TF 2.1 Bug: reset_states() does nothing in TF 2.1. + // the implementation below is how TF 2.1 should behave. + // to macth TF 2.1, just comment out the code below. if (is_stateful()) { forward_state_h_ = tensor(tensor_shape(n_units_), static_cast(0)); forward_state_c_ = tensor(tensor_shape(n_units_), static_cast(0)); diff --git a/test/stateful_test/stateful_recurrent_tests.cpp b/test/stateful_test/stateful_recurrent_tests.cpp index 05e321b0..dfeb359a 100644 --- a/test/stateful_test/stateful_recurrent_tests.cpp +++ b/test/stateful_test/stateful_recurrent_tests.cpp @@ -275,7 +275,7 @@ int main() vec_append(all_results, *result[0].as_vector()); // *********** TEST 16: "bidi-LSTM_stateful_init_state.json" *********** - model = load_model("./models/bidi-LSTM_nonstateful_init_state.json"); + model = load_model("./models/bidi-LSTM_stateful_init_state.json"); /// state_reset = true result = model.predict_stateful({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); vec_append(all_results, *result[0].as_vector()); diff --git a/test/stateful_test/stateful_recurrent_tests.py b/test/stateful_test/stateful_recurrent_tests.py index f0828a1f..d5940016 100644 --- a/test/stateful_test/stateful_recurrent_tests.py +++ b/test/stateful_test/stateful_recurrent_tests.py @@ -1,7 +1,7 @@ - -import os -os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' -os.environ['CUDA_VISIBLE_DEVICES']='' +# to hide any GPUs. +# import os +# os.environ['CUDA_DEVICE_ORDER']='PCI_BUS_ID' +# os.environ['CUDA_VISIBLE_DEVICES']='' import errno import os @@ -181,10 +181,6 @@ def main(): # Memory growth must be set before GPUs have been initialized print(e) - #### DEBUG.... - # rm old ./models if there and create new one. - os.system('rm -rf ./models') - os.makedirs('./models') # generate toy data train_seq_length = 4 feature_dim = 1 @@ -271,9 +267,5 @@ def main(): sys.exit(errno.EIO) print('\n\nPassed all stateful tests') - ## debug - all_results.to_file('models/keras_results.npy') - - if __name__ == "__main__": main() diff --git a/test/stateful_test/temp_old_bidi_full.py b/test/stateful_test/temp_old_bidi_full.py deleted file mode 100644 index 5419b8a4..00000000 --- a/test/stateful_test/temp_old_bidi_full.py +++ /dev/null @@ -1,100 +0,0 @@ -import h5py -import numpy as np -from tensorflow.keras.layers import Input, Dense, LSTM, Bidirectional -from tensorflow.keras import Model - -#### generate toy data -stream_seq_length = 4 -seq_ratio = 2 -train_seq_length = stream_seq_length * seq_ratio -feature_dim = 2 -num_seqs = 3 - -x = np.random.randint(0, high=2, size = (num_seqs * train_seq_length, feature_dim) ) -x = np.sign( x - 0.5 ) -y = np.sum( ( x == np.roll(x, 1, axis = 0) ), axis = 1 ) -### y[n] = number of agreements between x[n], x[n-1] -x = x.reshape( (num_seqs, train_seq_length, feature_dim) ) -y = y.reshape( (num_seqs, train_seq_length, 1) ) - - -###### Define/Build/Train Training Model -lstm_output_dim = 4 - -training_in_shape = x.shape[1:] -training_in = Input(shape=training_in_shape) -# training_in = Input(batch_shape=(None,train_seq_length,feature_dim)) this works too -foo = Bidirectional( LSTM(lstm_output_dim, return_sequences=True, stateful=False) ) (training_in) -training_pred = Dense(1)(foo) -training_model = Model(inputs=training_in, outputs=training_pred) -training_model.compile(loss='mean_squared_error', optimizer='adam') -training_model.summary() -training_model.fit(x, y, batch_size=2, epochs=10) - -training_model.save_weights('weights.hd5', overwrite=True) - -##### define the streaming-infernece model -streaming_in = Input(batch_shape=(1,stream_seq_length,feature_dim)) ## stateful ==> needs batch_shape specified -initial_state_h = Input(batch_shape=(1,lstm_output_dim)) -initial_state_c = Input(batch_shape=(1,lstm_output_dim)) - -fwd_initial_state_h = Input(batch_shape=(1,lstm_output_dim)) -fwd_initial_state_c = Input(batch_shape=(1,lstm_output_dim)) -bwd_initial_state_h = Input(batch_shape=(1,lstm_output_dim)) -bwd_initial_state_c = Input(batch_shape=(1,lstm_output_dim)) - - -lstm_streaming_stateful = Bidirectional( LSTM(lstm_output_dim, return_sequences=True, stateful=True ) ) (streaming_in) -lstm_streaming_initial_stateful = Bidirectional( LSTM(lstm_output_dim, return_sequences=True, stateful=True) ) (streaming_in, initial_state=[fwd_initial_state_h, fwd_initial_state_c, bwd_initial_state_h, bwd_initial_state_c]) -lstm_streaming_initial_nonstateful = Bidirectional( LSTM(lstm_output_dim, return_sequences=True, stateful=False) ) (streaming_in, initial_state=[fwd_initial_state_h, fwd_initial_state_c, bwd_initial_state_h, bwd_initial_state_c]) - - -streaming_stateful_model = Model( inputs=streaming_in, outputs=Dense(1)(lstm_streaming_stateful) ) -streaming_initial_stateful_model = Model( inputs=[streaming_in,fwd_initial_state_h, fwd_initial_state_c, bwd_initial_state_h, bwd_initial_state_c], outputs=Dense(1)(lstm_streaming_initial_stateful) ) -streaming_initial_nonstateful_model = Model( inputs=[streaming_in, fwd_initial_state_h, fwd_initial_state_c, bwd_initial_state_h, bwd_initial_state_c], outputs=Dense(1)(lstm_streaming_initial_nonstateful) ) - -streaming_stateful_model.compile(loss='mean_squared_error', optimizer='adam') -streaming_initial_stateful_model.compile(loss='mean_squared_error', optimizer='adam') -streaming_initial_nonstateful_model.compile(loss='mean_squared_error', optimizer='adam') - -streaming_stateful_model.load_weights('weights.hd5') -streaming_initial_stateful_model.load_weights('weights.hd5') -streaming_initial_nonstateful_model.load_weights('weights.hd5') - - -##### demo the behaivor - -fwd_initial_c = np.random.normal(0,1, (1,lstm_output_dim)) -fwd_initial_h = np.random.normal(0,1, (1,lstm_output_dim)) -bwd_initial_c = np.random.normal(0,1, (1,lstm_output_dim)) -bwd_initial_h = np.random.normal(0,1, (1,lstm_output_dim)) -# initial_c = np.zeros((1,lstm_output_dim)) -# initial_h = np.zeros((1,lstm_output_dim)) - -state_reset = True - -print('\n\n******the streaming-inference model can replicate the sequence-based trained model:\n') -for s in range(num_seqs): - if state_reset: - print(f'\n\nRunning Sequence {s} with STATE RESET:\n') - else: - print(f'\n\nRunning Sequence {s} with NO STATE RESET:\n') - in_seq = x[s].reshape( (1, train_seq_length, feature_dim) ) - seq_pred = training_model.predict(in_seq) - seq_pred = seq_pred.reshape(train_seq_length) - for k in range(seq_ratio): - in_feature_vector = x[s][ k * stream_seq_length : (k+1) * stream_seq_length ].reshape(1,stream_seq_length,feature_dim) - stream_stateful_pred = streaming_stateful_model.predict(in_feature_vector).reshape(stream_seq_length) - stream_stateful_init_pred = streaming_initial_stateful_model.predict([in_feature_vector, fwd_initial_c, fwd_initial_h, bwd_initial_c, bwd_initial_h]).reshape(stream_seq_length) - stream_nonstateful_init_pred = streaming_initial_nonstateful_model.predict([in_feature_vector, fwd_initial_c, fwd_initial_h, bwd_initial_c, bwd_initial_h]).reshape(stream_seq_length) - seq_pred_chunk = seq_pred[k * stream_seq_length : (k+1) * stream_seq_length] - for n in range(stream_seq_length): - msg = f'Seq-model Prediction[{k}, {n}]: { seq_pred_chunk[n] : 3.2f}\n' - msg += f'no-init-stateful, diff: {stream_stateful_pred[n] : 3.2f}, {seq_pred_chunk[n] - stream_stateful_pred[n] : 3.2f}\n' - msg += f'init-stateful, diff: {stream_stateful_init_pred[n] : 3.2f}, {seq_pred_chunk[n] - stream_stateful_init_pred[n] : 3.2f}\n' - msg += f'init-not-stateful, diff: {stream_nonstateful_init_pred[n] : 3.2f}, {seq_pred_chunk[n] - stream_nonstateful_init_pred[n] : 3.2f}\n\n' - print(msg) - if state_reset: - streaming_stateful_model.reset_states() - # streaming_initial_stateful_model.reset_states() <<----- this causes an error. - streaming_initial_nonstateful_model.reset_states() diff --git a/test/stateful_test/temp_rnn_results.txt b/test/stateful_test/temp_rnn_results.txt deleted file mode 100644 index 5f803302..00000000 --- a/test/stateful_test/temp_rnn_results.txt +++ /dev/null @@ -1,79 +0,0 @@ - -TF 2.0: ------------------------ -non_stateful: [-7.676248 -7.5144324 -7.3429837] -stateful: [-7.676248 -7.5144324 -7.3429837] -delta: [0. 0. 0.] -non_stateful: [-7.676248 -7.5144324 -7.3429837] -stateful: [-7.676248 -7.5144324 -7.3429837] -delta: [0. 0. 0.] - -** RESETING STATES in STATEFUL MODEL ** - -non_stateful: [-7.676248 -7.5144324 -7.3429837] -stateful: [-7.676248 -7.5144324 -7.3429837] -delta: [0. 0. 0.] -non_stateful: [-7.676248 -7.5144324 -7.3429837] -stateful: [-7.676248 -7.5144324 -7.3429837] -delta: [0. 0. 0.] - - -TF 2.1: ------------------------ -non_stateful: [-7.676248 -7.5144324 -7.3429837] -stateful: [-7.676248 -7.5144324 -7.3429837] -delta: [0. 0. 0.] -non_stateful: [-7.676248 -7.5144324 -7.3429837] -stateful: [-7.247089 -7.058707 -6.857616] -delta: [0.42915916 0.4557252 0.48536777] - -** RESETING STATES in STATEFUL MODEL ** - -non_stateful: [-7.676248 -7.5144324 -7.3429837] -stateful: [-7.676248 -7.5144324 -7.3429837] -delta: [0. 0. 0.] -non_stateful: [-7.676248 -7.5144324 -7.3429837] -stateful: [-7.247089 -7.058707 -6.857616] -delta: [0.42915916 0.4557252 0.48536777] - -FD starting point: ------------------------ -starting A -Non-Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Non-Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -** RESETING STATES in STATEFUL MODEL ** -Non-Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Non-Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] - -After Mods: --------------------------------------------- -starting A -Non-Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Non-Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Stateful -[[[[[[[-7.2471], [-7.0587], [-6.8576]]]]]]] -** RESETING STATES in STATEFUL MODEL ** -Non-Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Non-Stateful -[[[[[[[-7.6762], [-7.5144], [-7.3430]]]]]]] -Stateful -[[[[[[[-7.2471], [-7.0587], [-6.8576]]]]]]] \ No newline at end of file diff --git a/test/stateful_test/temp_stateful.h5 b/test/stateful_test/temp_stateful.h5 deleted file mode 100644 index 7ba9b45f7aebe3c47e6922685a6f04f8c83dad8a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13344 zcmeHNOK%)S5bm)_Sb~k#fD{J;3=2p>7-O$}uyTnJFh9OQ0>>DL|mE@a)iZDt8 zzqmX%CxDaXb4|ng0xEriZahs0H0C~BM0>B2(dc7n`D@XyheQKn1?9gWUKSN`%?a(uy6%LL8~8v!V;FKIMw8M(c`oZCpU9b!HY8?A zUPa1Vv0K+1zkQ$_8Mq_|p-A-_mU0vxt-$ZNUFZgRVqd*}JpoK4PlaecAD|2@1~8%C z{2n;2|2-+Y#owwD2gP;ctYR#mg@owp*dkDjYm+R!Nq)qJyZ%}dTa!Z7bA&yFIATNtk`jD)lz^*tM(d-T$&s+r^d|5CWvgu8DrhJB*OAV|$SUgwXDGopy%6y3zZrZ2?n}1|-*C2@=0;`Cy(@y9tc{TlGWJyI!KV z1mJ=6p1*vhfHfl^{Tln?&~|-$#X~IPFhMBG@D80g34JSazDXRvC6OX-?uqR<`#!pX zg)88?9kvlxeoYawbrLU~E%C#^!)IoR-JWCh?5~3`Lri=(MmgXXHM{s3jabiyFY zD=8HtOGBrdcy>sZqS`s7OS|Q)r9BdAH{P&Hdud8Y8Ig3f80^|LV_C19u&kpU25Uir zawa4`DcZ#zx3}gwJ#YthmAbSua=<@2v3$+w!r8pgkBVKd*yT724bPDV1FG@ zR3Ubri-AxzDD}w%U7_pNOjSRI&Fx$)hw{&voPwD4wVC_qV&R=2hkRUd)pp`bp)H z!Tx=~;+sko&vS#Ge|byMS+5;DOohUHx9D7RhxFwrsJ@=(o1y2crlKF0Pd=9};*jPS zyw@2O>V_*lm>w}N(0BD&G5x%lzDEbLd``U^A4WFcv)B3Yj;Z*ziCyO*0zy}%2<(-I zZkTy?j{C#PLmWTF{P4Krr~0x{9viOWU6r_!qZ_pXIDNa|!>F;Qo6JQ$35`XrXZa7D~~7SA7Uli%|Aq9135>ShK3 zh2sEaBOz|lej~$@YjT`dr*dTdAhs_62)BiporU@j_@Djj&mXbYMI)dQ&rv^dVLcVp_cgFl*v zC?ZFYK;ncr_Q{r9^F-zNdEI^US#h^RxXp#B%c%c3N%IH4U`SDi3&10U$83`5SuXi_#P&t-e$6*)7~MnsMD zm88B^yLrX&Tl?yfflGQ2ioRW=Qjfx;8Tf6t18mSI_O$EQV<1HORETEt0qVeJfDqcv z_krX3-{X=k?x;@e7uWUEO0akpQlh&fi$FfEP1@;o@+023>#rxVH7!(M6`e4VERLr+ z7JR^-<9Y>y7}^f4HzJlO4oASl$AxEaI-%v;T?ey?5HG=k&q8lNd}E`>Sg$8L$xOC# zulU|O9@FuSrgEIsPs!!Qn(BJL23TJdm%qHOM0R?TYOwI};sv=A3P821UJ4M^$?D|n z2XULzta?!pB6j>J2-{w;q4Y2gGJgPRtJ3uBD3Zuk&YG3aFfUgo%!*1yK-#Jxs>qW$ z3V9Oj2uREM>7U~5zkv55r(@NQn`@SWJ6f~XQQ`9RggG-|PB%bhE8di}3YcogGgX`( zt=nOQf^FN293X^tr{lCz1h$OsW@Q_M@-!g1{%Vl;Ez5_^NwXUu_`gj*FuSV-c1r*r z$nJ%UmvUG$0^rrii9_4LdA1^$Ce%y zI?W^u9Y40R5|P{K25t+tn?#Pa>e^|O;Gyrs+lK4Av1@zoci^4q*ICbB@PaUiFZpge z2)j8%BS*TOpLpIj`dHtLy>%{z@mkOcd^;af9k?a9)yXN)me-Ef?_=y&P8dX4Bc)-q z)6nT8o*mLo(d>-Tg>1QMAxA>(#+z0lmzIQl5lK&rVPCseEaJ)qix{miSPv4^GbQoq zqn+ot-F46Df;+IQROLP+2mGTQ%iD}ToK6cA6uUxec^T4#e0Bk>a97~p)^=mUWHJV1 zkKAY=izf85A|o;7oRVlH)J(WSwBbh5UvBP|pmZ7iljW4Rkt*St;kauk%`$jZ&p^<`7hkIFCCrSmxS^9!Eqj0<(b z)h|qsndj)6y(VU#7qj=MAj@^?-S{xFS{)3*M36atI`pGW4e@eKuOW9FMw3pd5dk~sN2EprTZhy0--r4Y}6u?-vL_vHZo z!Xn*Xq^B;@csMeUA0z;^2mPq=GJ>kp7k8OrCQ@* zMj)SWI7rWs|FG@Z7!_hr^rBB<%S~6r)Nj82aFhI&=jZ)6HB}E&2q+wfC>sfJgWfk% zBAFq_d3h#7))!*u9Ds0Bh*}N$KjDAv@4tS+UN?<^MnEH=5zq)|1db>IkMH~P^r8H| FybtKV{|Nv9 diff --git a/test/stateful_test/temp_stateless.json b/test/stateful_test/temp_stateless.json deleted file mode 100644 index 85273d22..00000000 --- a/test/stateful_test/temp_stateless.json +++ /dev/null @@ -1,184 +0,0 @@ -{ - "architecture": { - "backend": "tensorflow", - "class_name": "Model", - "config": { - "input_layers": [ - [ - "input_1", - 0, - 0 - ], - [ - "input_2", - 0, - 0 - ] - ], - "layers": [ - { - "class_name": "InputLayer", - "config": { - "batch_input_shape": [ - 1, - 3, - 1 - ], - "dtype": "float32", - "name": "input_1", - "sparse": false - }, - "inbound_nodes": [], - "name": "input_1" - }, - { - "class_name": "InputLayer", - "config": { - "batch_input_shape": [ - 1, - 1 - ], - "dtype": "float32", - "name": "input_2", - "sparse": false - }, - "inbound_nodes": [], - "name": "input_2" - }, - { - "class_name": "GRU", - "config": { - "activation": "linear", - "activity_regularizer": null, - "bias_constraint": null, - "bias_initializer": { - "class_name": "Zeros", - "config": {} - }, - "bias_regularizer": null, - "dropout": 0.0, - "dtype": "float32", - "go_backwards": false, - "implementation": 2, - "kernel_constraint": null, - "kernel_initializer": { - "class_name": "GlorotUniform", - "config": { - "seed": null - } - }, - "kernel_regularizer": null, - "name": "gru", - "recurrent_activation": "sigmoid", - "recurrent_constraint": null, - "recurrent_dropout": 0.0, - "recurrent_initializer": { - "class_name": "Orthogonal", - "config": { - "gain": 1.0, - "seed": null - } - }, - "recurrent_regularizer": null, - "reset_after": true, - "return_sequences": true, - "return_state": false, - "stateful": false, - "time_major": false, - "trainable": true, - "units": 1, - "unroll": false, - "use_bias": false - }, - "inbound_nodes": [ - [ - [ - "input_1", - 0, - 0, - {} - ], - [ - "input_2", - 0, - 0, - {} - ] - ] - ], - "name": "gru" - } - ], - "name": "model", - "output_layers": [ - [ - "gru", - 0, - 0 - ] - ] - }, - "keras_version": "2.2.4-tf" - }, - "hash": "9f36a6b98012df2ef10ea88aeb0589da9c1050a9c5875a64432e13e336e72536", - "image_data_format": "channels_last", - "input_shapes": [ - [ - 3, - 1 - ], - [ - 1 - ] - ], - "output_shapes": [ - [ - 3, - 1 - ] - ], - "tests": [ - { - "inputs": [ - { - "shape": [ - 3, - 1 - ], - "values": [ - "QYkivxu6Jb9bk3q/" - ] - }, - { - "shape": [ - 1 - ], - "values": [ - "o2GAvw==" - ] - } - ], - "outputs": [ - { - "shape": [ - 3, - 1 - ], - "values": [ - "CIylv9xzu7/NtgLA" - ] - } - ] - } - ], - "trainable_params": { - "gru": { - "recurrent_weights": [ - "AAAAvwAAAEDNzIy/" - ], - "weights": [ - "AACAPwAAAMAAAEBA" - ] - } - } -} diff --git a/test/stateful_test/temp_bidi_no_state_in.py b/test/stateful_test/tf_behaivor_scripts/temp_bidi_no_state_in.py similarity index 100% rename from test/stateful_test/temp_bidi_no_state_in.py rename to test/stateful_test/tf_behaivor_scripts/temp_bidi_no_state_in.py diff --git a/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.cpp b/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.cpp new file mode 100644 index 00000000..9bc58b0e --- /dev/null +++ b/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.cpp @@ -0,0 +1,72 @@ +#include "fdeep/fdeep.hpp" +#include // looks like we need this too (edit by π) + +using namespace fdeep; + +int main() +{ + + // x_in = np.random.normal(0,10,sequence_length) + // x_in = np.asarray([1,0,0]) + // x_in = x_in.reshape( (1, sequence_length, feature_dim) ) + + // fwd_initial_h = np.asarray(2.75).reshape(1,1) + // fwd_initial_c = np.asarray(1.3).reshape(1,1) + // bwd_initial_h = np.asarray(-2.0).reshape(1,1) + // bwd_initial_c = np.asarray(-1.2).reshape(1,1) + + const std::vector x_inf_0 = {1.0, 0.0, 0.0}; + const std::vector state_0 = {2.75}; + const std::vector state_1 = {1.3}; + const std::vector state_2 = {-2.0}; + const std::vector state_3 = {-1.2}; + + const shared_float_vec xt0(fplus::make_shared_ref(x_inf_0)); + const shared_float_vec st0(fplus::make_shared_ref(state_0)); + const shared_float_vec st1(fplus::make_shared_ref(state_1)); + const shared_float_vec st2(fplus::make_shared_ref(state_2)); + const shared_float_vec st3(fplus::make_shared_ref(state_3)); + + const tensor test_in_0(tensor_shape(3, 1), xt0); + const tensor test_state_0(tensor_shape(static_cast(1)), st0); + const tensor test_state_1(tensor_shape(static_cast(1)), st1); + const tensor test_state_2(tensor_shape(static_cast(1)), st2); + const tensor test_state_3(tensor_shape(static_cast(1)), st3); + + + std::cout << "loading models" << std::endl; + auto stateful_model = load_model("temp_stateful.json"); + auto stateless_model = load_model("temp_stateless.json"); + + // input for GRU: {test_in_0, test_state_0, test_state_2}; + // input for LSTM: {test_in_0, test_state_0, test_state_1, test_state_2, test_state_3} + + // A + std::cout << "starting A" << std::endl; + auto non_stateful_out = stateless_model.predict({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); + auto stateful_out = stateful_model.predict_stateful({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); + std::cout << "Non-Stateful" << std::endl; + std::cout << fdeep::show_tensors(non_stateful_out) << std::endl; + std::cout << "Stateful" << std::endl; + std::cout << fdeep::show_tensors(stateful_out) << std::endl; + + // B + std::cout << "starting B" << std::endl; + non_stateful_out = stateless_model.predict({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); + stateful_out = stateful_model.predict_stateful({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); + std::cout << "Non-Stateful" << std::endl; + std::cout << fdeep::show_tensors(non_stateful_out) << std::endl; + std::cout << "Stateful" << std::endl; + std::cout << fdeep::show_tensors(stateful_out) << std::endl; + + // C + std::cout << "starting C" << std::endl; + // stateful_model.reset_states(); + non_stateful_out = stateless_model.predict({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); + stateful_out = stateful_model.predict_stateful({test_in_0, test_state_0, test_state_1, test_state_2, test_state_3}); + std::cout << "Non-Stateful" << std::endl; + std::cout << fdeep::show_tensors(non_stateful_out) << std::endl; + std::cout << "Stateful" << std::endl; + std::cout << fdeep::show_tensors(stateful_out) << std::endl; +} + diff --git a/test/stateful_test/temp_bidi_state_in.py b/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.py similarity index 52% rename from test/stateful_test/temp_bidi_state_in.py rename to test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.py index ec0b2a76..2d51b5f0 100644 --- a/test/stateful_test/temp_bidi_state_in.py +++ b/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.py @@ -6,7 +6,7 @@ from tensorflow.keras.layers import Input, Dense, SimpleRNN, GRU, LSTM, Bidirectional from tensorflow.keras.models import Model -REC = SimpleRNN +REC = LSTM sequence_length = 3 feature_dim = 1 @@ -20,35 +20,45 @@ two_state_shape = [state_h_fwd_in, state_h_bwd_in] if REC == LSTM: - rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=False))(features_in, initial_state=four_state_shape) - stateful_rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=True))(features_in, initial_state=four_state_shape) + rnn_out = Bidirectional( REC(1, activation='linear', use_bias=False, return_sequences=True, return_state=False, stateful=False))(features_in, initial_state=four_state_shape) + stateful_rnn_out = Bidirectional( REC(1, activation='linear', use_bias=False, return_sequences=True, return_state=False, stateful=True))(features_in, initial_state=four_state_shape) rnn_inputs = [features_in, state_h_fwd_in, state_c_fwd_in, state_h_bwd_in, state_c_bwd_in] else: - rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=False))(features_in, initial_state=two_state_shape) - stateful_rnn_out = Bidirectional( REC(1, activation=None, use_bias=False, return_sequences=True, return_state=False, stateful=True))(features_in, initial_state=two_state_shape) + if REC == SimpleRNN: + rnn_out = Bidirectional( REC(1, activation='linear', use_bias=False, return_sequences=True, return_state=False, stateful=False))(features_in, initial_state=two_state_shape) + stateful_rnn_out = Bidirectional( REC(1, activation='linear', use_bias=False, return_sequences=True, return_state=False, stateful=True))(features_in, initial_state=two_state_shape) + else: + rnn_out = Bidirectional( REC(1, activation='linear', use_bias=False, return_sequences=True, return_state=False, stateful=False))(features_in, initial_state=two_state_shape) + stateful_rnn_out = Bidirectional( REC(1, activation='linear', use_bias=False, return_sequences=True, return_state=False, stateful=True))(features_in, initial_state=two_state_shape) rnn_inputs = [features_in, state_h_fwd_in, state_h_bwd_in] stateless_model = Model(inputs=rnn_inputs, outputs=rnn_out) stateful_model = Model(inputs=rnn_inputs, outputs=stateful_rnn_out) -toy_weights = [np.asarray([[ 1.0]], dtype=np.float32), np.asarray([[0.5 ]], dtype=np.float32), np.asarray([[ 0.0 ]], dtype=np.float32), np.asarray([[ 0.0 ]], dtype=np.float32)] -stateless_model.set_weights(toy_weights) -stateful_model.set_weights(toy_weights) +# toy_weights = [np.asarray([[ 1.0]], dtype=np.float32), np.asarray([[0.5 ]], dtype=np.float32), np.asarray([[ -1.0 ]], dtype=np.float32), np.asarray([[ -0.5 ]], dtype=np.float32)] +# stateless_model.set_weights(toy_weights) +# stateful_model.set_weights(toy_weights) -# stateful_model.set_weights( stateless_model.get_weights() ) +stateful_model.set_weights( stateless_model.get_weights() ) + +stateful_model.save('temp_stateful.h5') +stateless_model.save('temp_stateless.h5') x_in = np.random.normal(0,10,sequence_length) x_in = np.asarray([1,0,0]) x_in = x_in.reshape( (1, sequence_length, feature_dim) ) -fwd_initial_h = np.asarray(100.0).reshape(1,1) -fwd_initial_h = np.asarray(12.75).reshape(1,1) - +fwd_initial_h = np.asarray(2.75).reshape(1,1) +fwd_initial_c = np.asarray(1.3).reshape(1,1) +bwd_initial_h = np.asarray(-2.0).reshape(1,1) +bwd_initial_c = np.asarray(-1.2).reshape(1,1) -bwd_initial_h = np.asarray(-100.0).reshape(1,1) -fwd_initial_c = np.asarray(50.0).reshape(1,1) -bwd_initial_c = np.asarray(-50.0).reshape(1,1) +# fwd_initial_h = np.asarray(np.random.normal(0,10)).reshape(1,1) +# fwd_initial_h = np.asarray(np.random.normal(0,10)).reshape(1,1) +# bwd_initial_h = np.asarray(np.random.normal(0,10)).reshape(1,1) +# fwd_initial_c = np.asarray(np.random.normal(0,10)).reshape(1,1) +# bwd_initial_c = np.asarray(np.random.normal(0,10)).reshape(1,1) if REC == LSTM: rnn_input = [x_in, fwd_initial_h, fwd_initial_c, bwd_initial_h, bwd_initial_c] @@ -65,7 +75,6 @@ def print_bidi_out(non_stateful_out, stateful_out): print(f'stateful: {stateful_out.T[i]}') print(f'delta: {stateful_out.T[i]-non_stateful_out.T[i]}') - non_stateful_out = stateless_model.predict(rnn_input).reshape((sequence_length,2)) stateful_out = stateful_model.predict(rnn_input).reshape((sequence_length,2)) print_bidi_out(non_stateful_out, stateful_out) diff --git a/test/stateful_test/temp_rnn_test.cpp b/test/stateful_test/tf_behaivor_scripts/temp_rnn_test.cpp similarity index 100% rename from test/stateful_test/temp_rnn_test.cpp rename to test/stateful_test/tf_behaivor_scripts/temp_rnn_test.cpp diff --git a/test/stateful_test/temp_rnn_test.py b/test/stateful_test/tf_behaivor_scripts/temp_rnn_test.py similarity index 100% rename from test/stateful_test/temp_rnn_test.py rename to test/stateful_test/tf_behaivor_scripts/temp_rnn_test.py From 91653b68dfa9f12dbfb2c5873823d509d3346f9d Mon Sep 17 00:00:00 2001 From: Dobiasd Date: Fri, 10 Apr 2020 08:40:35 +0200 Subject: [PATCH 6/7] Minor cleanup --- include/fdeep/layers/bidirectional_layer.hpp | 8 ++++---- test/stateful_test/stateful_recurrent_tests.cpp | 2 +- .../tf_behaivor_scripts/temp_bidi_state_in.cpp | 2 +- test/stateful_test/tf_behaivor_scripts/temp_rnn_test.cpp | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/fdeep/layers/bidirectional_layer.hpp b/include/fdeep/layers/bidirectional_layer.hpp index 02aed313..4290fdff 100644 --- a/include/fdeep/layers/bidirectional_layer.hpp +++ b/include/fdeep/layers/bidirectional_layer.hpp @@ -64,9 +64,9 @@ class bidirectional_layer : public layer void reset_states() override { - // TF 2.1 Bug: reset_states() does nothing in TF 2.1. + // TF 2.1 Bug: reset_states() does nothing in TF 2.1. // the implementation below is how TF 2.1 should behave. - // to macth TF 2.1, just comment out the code below. + // to match TF 2.1, just comment out the code below. if (is_stateful()) { forward_state_h_ = tensor(tensor_shape(n_units_), static_cast(0)); forward_state_c_ = tensor(tensor_shape(n_units_), static_cast(0)); @@ -117,7 +117,7 @@ class bidirectional_layer : public layer "Invalid number of input tensors."); bool initial_state_provided = inputs.size() == 5; - bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; + bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state; // bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state; @@ -159,7 +159,7 @@ class bidirectional_layer : public layer "Invalid number of input tensors."); bool initial_state_provided = inputs.size() == 3; - bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; + bool use_last_state_for_initial_state = is_stateful() && !use_avail_input_state_for_stateful_; bool use_input_initial_state = initial_state_provided && !use_last_state_for_initial_state; // bool use_zero_initial_state = !use_input_initial_state && !use_last_state_for_initial_state; diff --git a/test/stateful_test/stateful_recurrent_tests.cpp b/test/stateful_test/stateful_recurrent_tests.cpp index dfeb359a..cc3a0f6b 100644 --- a/test/stateful_test/stateful_recurrent_tests.cpp +++ b/test/stateful_test/stateful_recurrent_tests.cpp @@ -1,5 +1,5 @@ #include "fdeep/fdeep.hpp" -#include // looks like we need this too (edit by π) +#include using namespace fdeep; diff --git a/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.cpp b/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.cpp index 9bc58b0e..3e266a9f 100644 --- a/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.cpp +++ b/test/stateful_test/tf_behaivor_scripts/temp_bidi_state_in.cpp @@ -1,5 +1,5 @@ #include "fdeep/fdeep.hpp" -#include // looks like we need this too (edit by π) +#include using namespace fdeep; diff --git a/test/stateful_test/tf_behaivor_scripts/temp_rnn_test.cpp b/test/stateful_test/tf_behaivor_scripts/temp_rnn_test.cpp index 732558d8..5e627b83 100644 --- a/test/stateful_test/tf_behaivor_scripts/temp_rnn_test.cpp +++ b/test/stateful_test/tf_behaivor_scripts/temp_rnn_test.cpp @@ -1,5 +1,5 @@ #include "fdeep/fdeep.hpp" -#include // looks like we need this too (edit by π) +#include using namespace fdeep; @@ -19,7 +19,7 @@ int main() std::cout << "loading models" << std::endl; auto stateful_model = load_model("temp_stateful.json"); auto stateless_model = load_model("temp_stateless.json"); - + // A std::cout << "starting A" << std::endl; auto non_stateful_out = stateless_model.predict({test_in_0, test_state_0}); From 37eaf671b1f6f3518c12697e1152f4770094cf57 Mon Sep 17 00:00:00 2001 From: Dobiasd Date: Fri, 10 Apr 2020 08:44:50 +0200 Subject: [PATCH 7/7] Bump version to 0.13.0 --- CMakeLists.txt | 2 +- INSTALL.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 81a83e28..d4f0560e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ list(APPEND CMAKE_MODULE_PATH "${FDEEP_TOP_DIR}/cmake") include(cmake/hunter.cmake) # default off -project(frugally-deep VERSION 0.12.1) +project(frugally-deep VERSION 0.13.0) message(STATUS "===( ${PROJECT_NAME} ${PROJECT_VERSION} )===") diff --git a/INSTALL.md b/INSTALL.md index 860af1bc..ed4c9262 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -63,7 +63,7 @@ Just add a *conanfile.txt* with frugally-deep as a requirement and chose the gen ``` [requires] -frugally-deep/v0.12.1-p0@dobiasd/stable +frugally-deep/v0.13.0-p0@dobiasd/stable [generators] cmake