From e748075e736837f4f1efa3f43acb24e9d7c437fb Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Thu, 19 Mar 2020 16:36:47 -0600 Subject: [PATCH 01/18] Removes pocketsphinx --- precise/pocketsphinx/__init__.py | 0 precise/pocketsphinx/listener.py | 69 -------------- precise/pocketsphinx/scripts/__init__.py | 0 precise/pocketsphinx/scripts/listen.py | 67 -------------- precise/pocketsphinx/scripts/test.py | 113 ----------------------- 5 files changed, 249 deletions(-) delete mode 100644 precise/pocketsphinx/__init__.py delete mode 100644 precise/pocketsphinx/listener.py delete mode 100644 precise/pocketsphinx/scripts/__init__.py delete mode 100755 precise/pocketsphinx/scripts/listen.py delete mode 100755 precise/pocketsphinx/scripts/test.py diff --git a/precise/pocketsphinx/__init__.py b/precise/pocketsphinx/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/precise/pocketsphinx/listener.py b/precise/pocketsphinx/listener.py deleted file mode 100644 index da20b43c..00000000 --- a/precise/pocketsphinx/listener.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Mycroft AI Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import numpy as np -from typing import * -from typing import BinaryIO - -from precise.params import pr -from precise.util import audio_to_buffer - - -class PocketsphinxListener: - """Pocketsphinx listener implementation used for comparison with Precise""" - - def __init__(self, key_phrase, dict_file, hmm_folder, threshold=1e-90, chunk_size=-1): - from pocketsphinx import Decoder - config = Decoder.default_config() - config.set_string('-hmm', hmm_folder) - config.set_string('-dict', dict_file) - config.set_string('-keyphrase', key_phrase) - config.set_float('-kws_threshold', float(threshold)) - config.set_float('-samprate', 16000) - config.set_int('-nfft', 2048) - config.set_string('-logfn', '/dev/null') - self.key_phrase = key_phrase - self.buffer = b'\0' * pr.sample_depth * pr.buffer_samples - self.pr = pr - self.read_size = -1 if chunk_size == -1 else pr.sample_depth * chunk_size - - try: - self.decoder = Decoder(config) - except RuntimeError: - options = dict(key_phrase=key_phrase, dict_file=dict_file, - hmm_folder=hmm_folder, threshold=threshold) - raise RuntimeError('Invalid Pocketsphinx options: ' + str(options)) - - def _transcribe(self, byte_data): - self.decoder.start_utt() - self.decoder.process_raw(byte_data, False, False) - self.decoder.end_utt() - return self.decoder.hyp() - - def found_wake_word(self, frame_data): - hyp = self._transcribe(frame_data + b'\0' * int(2 * 16000 * 0.01)) - return bool(hyp and self.key_phrase in hyp.hypstr.lower()) - - def update(self, stream: Union[BinaryIO, np.ndarray, bytes]) -> float: - if isinstance(stream, np.ndarray): - chunk = audio_to_buffer(stream) - else: - if isinstance(stream, (bytes, bytearray)): - chunk = stream - else: - chunk = stream.read(self.read_size) - if len(chunk) == 0: - raise EOFError - self.buffer = self.buffer[len(chunk):] + chunk - return float(self.found_wake_word(self.buffer)) diff --git a/precise/pocketsphinx/scripts/__init__.py b/precise/pocketsphinx/scripts/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/precise/pocketsphinx/scripts/listen.py b/precise/pocketsphinx/scripts/listen.py deleted file mode 100755 index 0403fc86..00000000 --- a/precise/pocketsphinx/scripts/listen.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Mycroft AI Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from precise_runner import PreciseRunner -from precise_runner.runner import ListenerEngine -from prettyparse import Usage -from threading import Event - -from precise.pocketsphinx.listener import PocketsphinxListener -from precise.scripts.base_script import BaseScript -from precise.util import activate_notify - - -class PocketsphinxListenScript(BaseScript): - usage = Usage(''' - Run Pocketsphinx on microphone audio input - - :key_phrase str - Key phrase composed of words from dictionary - - :dict_file str - Filename of dictionary with word pronunciations - - :hmm_folder str - Folder containing hidden markov model - - :-th --threshold str 1e-90 - Threshold for activations - - :-c --chunk-size int 2048 - Samples between inferences - ''') - - def run(self): - def on_activation(): - activate_notify() - - def on_prediction(conf): - print('!' if conf > 0.5 else '.', end='', flush=True) - - args = self.args - runner = PreciseRunner( - ListenerEngine( - PocketsphinxListener( - args.key_phrase, args.dict_file, args.hmm_folder, args.threshold, args.chunk_size - ) - ), 3, on_activation=on_activation, on_prediction=on_prediction - ) - runner.start() - Event().wait() # Wait forever - - -main = PocketsphinxListenScript.run_main - -if __name__ == '__main__': - main() diff --git a/precise/pocketsphinx/scripts/test.py b/precise/pocketsphinx/scripts/test.py deleted file mode 100755 index c782ce83..00000000 --- a/precise/pocketsphinx/scripts/test.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Mycroft AI Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import wave -from prettyparse import Usage -from subprocess import check_output, PIPE - -from precise.pocketsphinx.listener import PocketsphinxListener -from precise.scripts.base_script import BaseScript -from precise.scripts.test import Stats -from precise.train_data import TrainData - - -class PocketsphinxTestScript(BaseScript): - usage = Usage(''' - Test a dataset using Pocketsphinx - - :key_phrase str - Key phrase composed of words from dictionary - - :dict_file str - Filename of dictionary with word pronunciations - - :hmm_folder str - Folder containing hidden markov model - - :-th --threshold str 1e-90 - Threshold for activations - - :-t --use-train - Evaluate training data instead of test data - - :-nf --no-filenames - Don't show the names of files that failed - - ... - ''') | TrainData.usage - - def __init__(self, args): - super().__init__(args) - self.listener = PocketsphinxListener( - args.key_phrase, args.dict_file, args.hmm_folder, args.threshold - ) - - self.outputs = [] - self.targets = [] - self.filenames = [] - - def get_stats(self): - return Stats(self.outputs, self.targets, self.filenames) - - def run(self): - args = self.args - data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder) - print('Data:', data) - - ww_files, nww_files = data.train_files if args.use_train else data.test_files - self.run_test(ww_files, 'Wake Word', 1.0) - self.run_test(nww_files, 'Not Wake Word', 0.0) - stats = self.get_stats() - if not self.args.no_filenames: - fp_files = stats.calc_filenames(False, True, 0.5) - fn_files = stats.calc_filenames(False, False, 0.5) - print('=== False Positives ===') - print('\n'.join(fp_files)) - print() - print('=== False Negatives ===') - print('\n'.join(fn_files)) - print() - print(stats.counts_str(0.5)) - print() - print(stats.summary_str(0.5)) - - def eval_file(self, filename) -> float: - transcription = check_output( - ['pocketsphinx_continuous', '-kws_threshold', '1e-20', '-keyphrase', 'hey my craft', - '-infile', filename], stderr=PIPE) - return float(bool(transcription) and not transcription.isspace()) - - def run_test(self, test_files, label_name, label): - print() - print('===', label_name, '===') - for test_file in test_files: - try: - with wave.open(test_file) as wf: - frames = wf.readframes(wf.getnframes()) - except (OSError, EOFError): - print('?', end='', flush=True) - continue - - out = int(self.listener.found_wake_word(frames)) - self.outputs.append(out) - self.targets.append(label) - self.filenames.append(test_file) - print('!' if out else '.', end='', flush=True) - print() - - -main = PocketsphinxTestScript.run_main - -if __name__ == '__main__': - main() From 3df8f7add87ac452df30ee767ad098825a5377ea Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Wed, 25 Mar 2020 14:03:10 -0600 Subject: [PATCH 02/18] Convert all to tf2 using tf_upgrade_v2. --- precise/network_runner.py | 14 +++++++------- precise/scripts/add_noise.py | 0 precise/scripts/engine.py | 0 precise/scripts/eval.py | 0 precise/scripts/graph.py | 0 precise/scripts/listen.py | 0 precise/scripts/test.py | 0 precise/scripts/train_generated.py | 6 +++--- precise/scripts/train_incremental.py | 2 +- precise/scripts/train_optimize.py | 2 +- precise/scripts/train_sampled.py | 0 11 files changed, 12 insertions(+), 12 deletions(-) mode change 100755 => 100644 precise/scripts/add_noise.py mode change 100755 => 100644 precise/scripts/engine.py mode change 100755 => 100644 precise/scripts/eval.py mode change 100755 => 100644 precise/scripts/graph.py mode change 100755 => 100644 precise/scripts/listen.py mode change 100755 => 100644 precise/scripts/test.py mode change 100755 => 100644 precise/scripts/train_incremental.py mode change 100755 => 100644 precise/scripts/train_sampled.py diff --git a/precise/network_runner.py b/precise/network_runner.py index 2283b9db..a5680ca5 100644 --- a/precise/network_runner.py +++ b/precise/network_runner.py @@ -41,15 +41,15 @@ def __init__(self, model_name: str): print('Warning: ', model_name, 'looks like a Keras model.') self.tf = import_module('tensorflow') self.graph = self.load_graph(model_name) + with self.graph.as_default(): + self.inp_var = self.graph.get_operation_by_name('import/net_input').outputs[0] + self.out_var = self.graph.get_operation_by_name('import/net_output').outputs[0] - self.inp_var = self.graph.get_operation_by_name('import/net_input').outputs[0] - self.out_var = self.graph.get_operation_by_name('import/net_output').outputs[0] - - self.sess = self.tf.Session(graph=self.graph) + self.sess = self.tf.compat.v1.Session(graph=self.graph) def load_graph(self, model_file: str) -> 'tf.Graph': graph = self.tf.Graph() - graph_def = self.tf.GraphDef() + graph_def = self.tf.compat.v1.GraphDef() with open(model_file, "rb") as f: graph_def.ParseFromString(f.read()) @@ -71,10 +71,10 @@ def __init__(self, model_name: str): import tensorflow as tf # ISSUE 88 - Following 3 lines added to resolve issue 88 - JM 2020-02-04 per liny90626 from tensorflow.python.keras.backend import set_session # ISSUE 88 - self.sess = tf.Session() # ISSUE 88 + self.sess = tf.compat.v1.Session() # ISSUE 88 set_session(self.sess) # ISSUE 88 self.model = load_precise_model(model_name) - self.graph = tf.get_default_graph() + self.graph = tf.compat.v1.get_default_graph() def predict(self, inputs: np.ndarray): from tensorflow.python.keras.backend import set_session # ISSUE 88 diff --git a/precise/scripts/add_noise.py b/precise/scripts/add_noise.py old mode 100755 new mode 100644 diff --git a/precise/scripts/engine.py b/precise/scripts/engine.py old mode 100755 new mode 100644 diff --git a/precise/scripts/eval.py b/precise/scripts/eval.py old mode 100755 new mode 100644 diff --git a/precise/scripts/graph.py b/precise/scripts/graph.py old mode 100755 new mode 100644 diff --git a/precise/scripts/listen.py b/precise/scripts/listen.py old mode 100755 new mode 100644 diff --git a/precise/scripts/test.py b/precise/scripts/test.py old mode 100755 new mode 100644 diff --git a/precise/scripts/train_generated.py b/precise/scripts/train_generated.py index 9c275dfe..4aa62942 100644 --- a/precise/scripts/train_generated.py +++ b/precise/scripts/train_generated.py @@ -18,7 +18,7 @@ import numpy as np from contextlib import suppress from fitipy import Fitipy -from keras.callbacks import LambdaCallback +from tensorflow.keras.callbacks import LambdaCallback from os.path import splitext, join, basename from prettyparse import Usage from random import random, shuffle @@ -90,7 +90,7 @@ def __init__(self, args): self.model = create_model(args.model, params) self.listener = Listener('', args.chunk_size, runner_cls=lambda x: None) - from keras.callbacks import ModelCheckpoint, TensorBoard + from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard checkpoint = ModelCheckpoint(args.model, monitor=args.metric_monitor, save_best_only=args.save_best) epoch_fiti = Fitipy(splitext(args.model)[0] + '.epoch') @@ -234,7 +234,7 @@ def run(self): callbacks=self.callbacks, initial_epoch=self.epoch ) finally: - self.model.save(self.args.model) + self.model.save(self.args.model, save_format='h5') save_params(self.args.model) diff --git a/precise/scripts/train_incremental.py b/precise/scripts/train_incremental.py old mode 100755 new mode 100644 index d0238cef..ef2082d7 --- a/precise/scripts/train_incremental.py +++ b/precise/scripts/train_incremental.py @@ -107,7 +107,7 @@ def retrain(self): validation_data=test_data, callbacks=self.callbacks, initial_epoch=self.epoch ) finally: - self.listener.runner.model.save(self.args.model) + self.listener.runner.model.save(self.args.model,save_format='h5') def train_on_audio(self, fn: str): """Run through a single audio file""" diff --git a/precise/scripts/train_optimize.py b/precise/scripts/train_optimize.py index fc1bee6b..2ceaa55f 100644 --- a/precise/scripts/train_optimize.py +++ b/precise/scripts/train_optimize.py @@ -52,7 +52,7 @@ def __init__(self, args): data = TrainData.from_both(self.args.tags_file, self.args.tags_folder, self.args.folder) _, self.test = data.load(False, True) - from keras.callbacks import ModelCheckpoint + from tensorflow.keras.callbacks import ModelCheckpoint for i in list(self.callbacks): if isinstance(i, ModelCheckpoint): self.callbacks.remove(i) diff --git a/precise/scripts/train_sampled.py b/precise/scripts/train_sampled.py old mode 100755 new mode 100644 From 317a091a9da9460b543bf9fe07aa5238f4b7f0f9 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Wed, 25 Mar 2020 14:05:04 -0600 Subject: [PATCH 03/18] Upgrade TF to 2.1, Keras to 2.3.1 and remove pocketsphinx. --- setup.py | 8 +++----- setup.sh | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index a55da052..105910ea 100644 --- a/setup.py +++ b/setup.py @@ -44,9 +44,7 @@ ], packages=[ 'precise', - 'precise.scripts', - 'precise.pocketsphinx', - 'precise.pocketsphinx.scripts' + 'precise.scripts' ], entry_points={ 'console_scripts': [ @@ -71,10 +69,10 @@ }, install_requires=[ 'numpy', - 'tensorflow>=1.13,<1.14', # Must be on piwheels + 'tensorflow-gpu>=2', # Must be on piwheels 'sonopy', 'pyaudio', - 'keras<=2.1.5', + 'keras>2.1.5', 'h5py', 'wavio', 'typing', diff --git a/setup.sh b/setup.sh index 1c8a6a0b..c72c1b87 100755 --- a/setup.sh +++ b/setup.sh @@ -60,4 +60,4 @@ fi pip install -e runner/ pip install -e . -pip install pocketsphinx # Optional, for comparison +#pip install pocketsphinx # Optional, for comparison From 920641aec48b6134d365a02e40c6015c0b46d5a6 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Wed, 25 Mar 2020 14:13:25 -0600 Subject: [PATCH 04/18] Adjusts training parameters for 20 units in GRU. Adds early stop, reduce learning rate. Removes Tensorboard due to compat issues. --- precise/scripts/train.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) mode change 100755 => 100644 precise/scripts/train.py diff --git a/precise/scripts/train.py b/precise/scripts/train.py old mode 100755 new mode 100644 index b89b044b..c8caee9c --- a/precise/scripts/train.py +++ b/precise/scripts/train.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from fitipy import Fitipy -from keras.callbacks import LambdaCallback +from tensorflow.keras.callbacks import LambdaCallback from os.path import splitext, isfile from prettyparse import Usage from typing import Any, Tuple @@ -85,9 +85,11 @@ def __init__(self, args): self.model = create_model(args.model, params) self.train, self.test = self.load_data(self.args) - from keras.callbacks import ModelCheckpoint, TensorBoard + from keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping, ReduceLROnPlateau checkpoint = ModelCheckpoint(args.model, monitor=args.metric_monitor, save_best_only=args.save_best) + earlyStop = EarlyStopping(monitor='loss', min_delta=0.00005, patience=2500, verbose=1, mode='auto', baseline=None, restore_best_weights=True) + reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.85, min_delta=0.0001, patience=100, min_lr=0.0000005, verbose=1) epoch_fiti = Fitipy(splitext(args.model)[0] + '.epoch') self.epoch = epoch_fiti.read().read(0, int) @@ -104,9 +106,13 @@ def on_epoch_end(_a, _b): self.hash_to_ind = {} self.callbacks = [ - checkpoint, TensorBoard( - log_dir=self.model_base + '.logs', - ), LambdaCallback(on_epoch_end=on_epoch_end) + checkpoint, + # TensorBoard( # Disables tensorboard due to compat with tf2 + # log_dir=self.model_base + '.logs', + # ), + LambdaCallback(on_epoch_end=on_epoch_end), + reduce_lr, + earlyStop ] @staticmethod @@ -160,7 +166,9 @@ def run(self): self.model.fit( train_inputs, train_outputs, self.args.batch_size, self.epoch + self.args.epochs, validation_data=self.test, - initial_epoch=self.epoch, callbacks=self.callbacks + initial_epoch=self.epoch, callbacks=self.callbacks, + use_multiprocessing=True, validation_freq=5, + verbose=1 ) From 285499e3637c7bf897c736f3b8dbbea93d114a5f Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Mon, 30 Mar 2020 10:19:29 -0600 Subject: [PATCH 05/18] Adds TFLiteRunner with enough compatility for inference. --- precise/network_runner.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/precise/network_runner.py b/precise/network_runner.py index a5680ca5..550061e9 100644 --- a/precise/network_runner.py +++ b/precise/network_runner.py @@ -85,6 +85,34 @@ def predict(self, inputs: np.ndarray): def run(self, inp: np.ndarray) -> float: return self.predict(inp[np.newaxis])[0][0] +class TFLiteRunner(Runner): + def __init__(self, model_name: str): + import tensorflow as tf + # Setup tflite environment + self.interpreter = tf.lite.Interpreter(model_path=model_name) + self.interpreter.allocate_tensors() + + self.input_details = self.interpreter.get_input_details() + self.output_details = self.interpreter.get_output_details() + + def predict(self, inputs: np.ndarray): + # Format output to match Keras's model.predict output + output_data = np.ndarray((inputs.shape[0],1), dtype=np.float32) + + # Support for multiple inputs + for input in inputs: + # Format as float32. Add a wrapper dimension. + current = np.array([input]).astype(np.float32) + + # Load data, run inference and extract output from tensor + self.interpreter.set_tensor(self.input_details[0]['index'], current) + self.interpreter.invoke() + output_data[count] = self.interpreter.get_tensor(self.output_details[0]['index']) + + return output_data + + def run(self, inp: np.ndarray) -> float: + return self.predict(inp[np.newaxis])[0][0] class Listener: """Listener that preprocesses audio into MFCC vectors and executes neural networks""" @@ -102,7 +130,8 @@ def __init__(self, model_name: str, chunk_size: int = -1, runner_cls: type = Non def find_runner(model_name: str) -> Type[Runner]: runners = { '.net': KerasRunner, - '.pb': TensorFlowRunner + '.pb': TensorFlowRunner, + '.tflite': TFLiteRunner } ext = splitext(model_name)[-1] if ext not in runners: From eeaebf4ec5c278d1532a149b357a3f02879c5f58 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Thu, 2 Apr 2020 08:34:52 -0600 Subject: [PATCH 06/18] Bugfix: Increments counter for each prediction. --- precise/network_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/precise/network_runner.py b/precise/network_runner.py index 550061e9..d74eb8b7 100644 --- a/precise/network_runner.py +++ b/precise/network_runner.py @@ -97,6 +97,7 @@ def __init__(self, model_name: str): def predict(self, inputs: np.ndarray): # Format output to match Keras's model.predict output + count = 0 output_data = np.ndarray((inputs.shape[0],1), dtype=np.float32) # Support for multiple inputs @@ -108,6 +109,7 @@ def predict(self, inputs: np.ndarray): self.interpreter.set_tensor(self.input_details[0]['index'], current) self.interpreter.invoke() output_data[count] = self.interpreter.get_tensor(self.output_details[0]['index']) + count += 1 return output_data From 8f32394c0e652f7261a45dda625d718b59d99b69 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Thu, 2 Apr 2020 08:38:23 -0600 Subject: [PATCH 07/18] Adjusts KerasRunner for compat with tf2. --- precise/network_runner.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/precise/network_runner.py b/precise/network_runner.py index d74eb8b7..78907076 100644 --- a/precise/network_runner.py +++ b/precise/network_runner.py @@ -68,19 +68,17 @@ def run(self, inp: np.ndarray) -> float: class KerasRunner(Runner): def __init__(self, model_name: str): - import tensorflow as tf - # ISSUE 88 - Following 3 lines added to resolve issue 88 - JM 2020-02-04 per liny90626 - from tensorflow.python.keras.backend import set_session # ISSUE 88 - self.sess = tf.compat.v1.Session() # ISSUE 88 - set_session(self.sess) # ISSUE 88 + # Load model using Keras (not tf.keras) self.model = load_precise_model(model_name) - self.graph = tf.compat.v1.get_default_graph() + + # TF 2.0 doesn't work well with sessions and graphs + # Only in tf.v1.compat, but that restricts usage of v2 features + self.graph = None def predict(self, inputs: np.ndarray): - from tensorflow.python.keras.backend import set_session # ISSUE 88 - with self.graph.as_default(): - set_session(self.sess) # ISSUE 88 - return self.model.predict(inputs) + import keras as K + K.backend.tensorflow_backend._SYMBOLIC_SCOPE.value = True + return self.model.predict(inputs) def run(self, inp: np.ndarray) -> float: return self.predict(inp[np.newaxis])[0][0] From 15a5bf42cdfa176107c5d1e133e2cdd82af3f445 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Thu, 2 Apr 2020 08:39:44 -0600 Subject: [PATCH 08/18] Sets TF version to 2.2.0rc2. Update when 2.2.0 is released. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 105910ea..9035431e 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ }, install_requires=[ 'numpy', - 'tensorflow-gpu>=2', # Must be on piwheels + 'tensorflow-gpu==2.2.0rc', # This should be changed for 2.2.0 when it's released. 'sonopy', 'pyaudio', 'keras>2.1.5', From 51fc5627aef02e7a517fec2c7b0673ad7d3d463e Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Thu, 2 Apr 2020 08:48:30 -0600 Subject: [PATCH 09/18] Adapts convert script for tf2 and tflite. --- precise/scripts/convert.py | 56 ++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 33 deletions(-) mode change 100755 => 100644 precise/scripts/convert.py diff --git a/precise/scripts/convert.py b/precise/scripts/convert.py old mode 100755 new mode 100644 index d3bacb43..212813e4 --- a/precise/scripts/convert.py +++ b/precise/scripts/convert.py @@ -20,7 +20,6 @@ from precise.scripts.base_script import BaseScript - class ConvertScript(BaseScript): usage = Usage(''' Convert wake word model from Keras to TensorFlow @@ -46,42 +45,33 @@ def convert(self, model_path: str, out_file: str): out_file: location to write protobuf """ print('Converting', model_path, 'to', out_file, '...') - - import tensorflow as tf + + import tensorflow as tf # Using tensorflow v2.0 + from tensorflow import keras as K from precise.model import load_precise_model - from keras import backend as K - + from precise.functions import weighted_log_loss + out_dir, filename = split(out_file) out_dir = out_dir or '.' os.makedirs(out_dir, exist_ok=True) - - K.set_learning_phase(0) - model = load_precise_model(model_path) - - out_name = 'net_output' - tf.identity(model.output, name=out_name) - print('Output node name:', out_name) - print('Output folder:', out_dir) - - sess = K.get_session() - - # Write the graph in human readable - tf.train.write_graph(sess.graph.as_graph_def(), out_dir, filename + 'txt', as_text=True) - print('Saved readable graph to:', filename + 'txt') - - # Write the graph in binary .pb file - from tensorflow.python.framework import graph_util - from tensorflow.python.framework import graph_io - - cgraph = graph_util.convert_variables_to_constants(sess, sess.graph.as_graph_def(), [out_name]) - graph_io.write_graph(cgraph, out_dir, filename, as_text=False) - - if isfile(model_path + '.params'): - copyfile(model_path + '.params', out_file + '.params') - - print('Saved graph to:', filename) - - del sess + + # Load custom loss function with model + model = K.models.load_model(model_path, custom_objects={'weighted_log_loss': weighted_log_loss}) + + model.summary() + + # Support for freezing models to .pb has been removed in TF 2.0. + + # Converting instead to TFLite model + print('Starting TFLite conversion.') + converter = tf.lite.TFLiteConverter.from_keras_model(model) + converter.target_ops = [tf.lite.OpsSet.TFLITE_BUILTINS,tf.lite.OpsSet.SELECT_TF_OPS] + # converter.experimental_new_converter=True # TF2.2 uses experimental converter by default. + tflite_model = converter.convert() + open(out_file, "wb").write(tflite_model) + print('Wrote to ' + out_file) + + main = ConvertScript.run_main From 7d1ad8d2a6efa8632ea750ac496291fb5432b742 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Mon, 20 Apr 2020 08:49:44 -0500 Subject: [PATCH 10/18] Changes precise-convert default output file extension to tflite. --- precise/scripts/convert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/precise/scripts/convert.py b/precise/scripts/convert.py index 212813e4..420a19bf 100644 --- a/precise/scripts/convert.py +++ b/precise/scripts/convert.py @@ -27,8 +27,8 @@ class ConvertScript(BaseScript): :model str Input Keras model (.net) - :-o --out str {model}.pb - Custom output TensorFlow protobuf filename + :-o --out str {model}.tflite + Custom output TensorFlow Lite filename ''') def run(self): @@ -38,11 +38,11 @@ def run(self): def convert(self, model_path: str, out_file: str): """ - Converts an HD5F file from Keras to a .pb for use with TensorFlow + Converts an HD5F file from Keras to a .tflite for use with TensorFlow Runtime Args: model_path: location of Keras model - out_file: location to write protobuf + out_file: location to write TFLite model """ print('Converting', model_path, 'to', out_file, '...') From 1c3fe878b1fa9c7c45c1a1374e97183ee894daa6 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Mon, 20 Apr 2020 09:14:54 -0500 Subject: [PATCH 11/18] General cleanup for merge. --- precise/scripts/convert.py | 18 +++++++----------- precise/scripts/train.py | 11 ++--------- precise/scripts/train_generated.py | 9 ++++----- 3 files changed, 13 insertions(+), 25 deletions(-) diff --git a/precise/scripts/convert.py b/precise/scripts/convert.py index 420a19bf..cece7b2b 100644 --- a/precise/scripts/convert.py +++ b/precise/scripts/convert.py @@ -45,34 +45,30 @@ def convert(self, model_path: str, out_file: str): out_file: location to write TFLite model """ print('Converting', model_path, 'to', out_file, '...') - - import tensorflow as tf # Using tensorflow v2.0 + + import tensorflow as tf # Using tensorflow v2.2 from tensorflow import keras as K from precise.model import load_precise_model from precise.functions import weighted_log_loss - + out_dir, filename = split(out_file) out_dir = out_dir or '.' os.makedirs(out_dir, exist_ok=True) - + # Load custom loss function with model model = K.models.load_model(model_path, custom_objects={'weighted_log_loss': weighted_log_loss}) - model.summary() - - # Support for freezing models to .pb has been removed in TF 2.0. - + + # Support for freezing Keras models to .pb has been removed in TF 2.0. + # Converting instead to TFLite model print('Starting TFLite conversion.') converter = tf.lite.TFLiteConverter.from_keras_model(model) converter.target_ops = [tf.lite.OpsSet.TFLITE_BUILTINS,tf.lite.OpsSet.SELECT_TF_OPS] - # converter.experimental_new_converter=True # TF2.2 uses experimental converter by default. tflite_model = converter.convert() open(out_file, "wb").write(tflite_model) print('Wrote to ' + out_file) - - main = ConvertScript.run_main diff --git a/precise/scripts/train.py b/precise/scripts/train.py index c8caee9c..bbd2aa3d 100644 --- a/precise/scripts/train.py +++ b/precise/scripts/train.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from fitipy import Fitipy -from tensorflow.keras.callbacks import LambdaCallback +from keras.callbacks import LambdaCallback from os.path import splitext, isfile from prettyparse import Usage from typing import Any, Tuple @@ -85,11 +85,9 @@ def __init__(self, args): self.model = create_model(args.model, params) self.train, self.test = self.load_data(self.args) - from keras.callbacks import ModelCheckpoint, TensorBoard, EarlyStopping, ReduceLROnPlateau + from keras.callbacks import ModelCheckpoint checkpoint = ModelCheckpoint(args.model, monitor=args.metric_monitor, save_best_only=args.save_best) - earlyStop = EarlyStopping(monitor='loss', min_delta=0.00005, patience=2500, verbose=1, mode='auto', baseline=None, restore_best_weights=True) - reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.85, min_delta=0.0001, patience=100, min_lr=0.0000005, verbose=1) epoch_fiti = Fitipy(splitext(args.model)[0] + '.epoch') self.epoch = epoch_fiti.read().read(0, int) @@ -107,12 +105,7 @@ def on_epoch_end(_a, _b): self.callbacks = [ checkpoint, - # TensorBoard( # Disables tensorboard due to compat with tf2 - # log_dir=self.model_base + '.logs', - # ), LambdaCallback(on_epoch_end=on_epoch_end), - reduce_lr, - earlyStop ] @staticmethod diff --git a/precise/scripts/train_generated.py b/precise/scripts/train_generated.py index 4aa62942..715c1cf8 100644 --- a/precise/scripts/train_generated.py +++ b/precise/scripts/train_generated.py @@ -18,7 +18,7 @@ import numpy as np from contextlib import suppress from fitipy import Fitipy -from tensorflow.keras.callbacks import LambdaCallback +from keras.callbacks import LambdaCallback from os.path import splitext, join, basename from prettyparse import Usage from random import random, shuffle @@ -90,7 +90,7 @@ def __init__(self, args): self.model = create_model(args.model, params) self.listener = Listener('', args.chunk_size, runner_cls=lambda x: None) - from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard + from keras.callbacks import ModelCheckpoint checkpoint = ModelCheckpoint(args.model, monitor=args.metric_monitor, save_best_only=args.save_best) epoch_fiti = Fitipy(splitext(args.model)[0] + '.epoch') @@ -103,9 +103,8 @@ def on_epoch_end(_a, _b): self.model_base = splitext(self.args.model)[0] self.callbacks = [ - checkpoint, TensorBoard( - log_dir=self.model_base + '.logs', - ), LambdaCallback(on_epoch_end=on_epoch_end) + checkpoint, + LambdaCallback(on_epoch_end=on_epoch_end) ] self.data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder) From 26e6d25ffa84cf498f5b639a6dd154f139b6b5e6 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Tue, 21 Apr 2020 08:25:58 -0500 Subject: [PATCH 12/18] Restore pocketsphinx files. --- precise/pocketsphinx/__init__.py | 0 precise/pocketsphinx/listener.py | 69 ++++++++++++++ precise/pocketsphinx/scripts/__init__.py | 0 precise/pocketsphinx/scripts/listen.py | 67 ++++++++++++++ precise/pocketsphinx/scripts/test.py | 113 +++++++++++++++++++++++ 5 files changed, 249 insertions(+) create mode 100644 precise/pocketsphinx/__init__.py create mode 100644 precise/pocketsphinx/listener.py create mode 100644 precise/pocketsphinx/scripts/__init__.py create mode 100755 precise/pocketsphinx/scripts/listen.py create mode 100755 precise/pocketsphinx/scripts/test.py diff --git a/precise/pocketsphinx/__init__.py b/precise/pocketsphinx/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/precise/pocketsphinx/listener.py b/precise/pocketsphinx/listener.py new file mode 100644 index 00000000..da20b43c --- /dev/null +++ b/precise/pocketsphinx/listener.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# Copyright 2019 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import numpy as np +from typing import * +from typing import BinaryIO + +from precise.params import pr +from precise.util import audio_to_buffer + + +class PocketsphinxListener: + """Pocketsphinx listener implementation used for comparison with Precise""" + + def __init__(self, key_phrase, dict_file, hmm_folder, threshold=1e-90, chunk_size=-1): + from pocketsphinx import Decoder + config = Decoder.default_config() + config.set_string('-hmm', hmm_folder) + config.set_string('-dict', dict_file) + config.set_string('-keyphrase', key_phrase) + config.set_float('-kws_threshold', float(threshold)) + config.set_float('-samprate', 16000) + config.set_int('-nfft', 2048) + config.set_string('-logfn', '/dev/null') + self.key_phrase = key_phrase + self.buffer = b'\0' * pr.sample_depth * pr.buffer_samples + self.pr = pr + self.read_size = -1 if chunk_size == -1 else pr.sample_depth * chunk_size + + try: + self.decoder = Decoder(config) + except RuntimeError: + options = dict(key_phrase=key_phrase, dict_file=dict_file, + hmm_folder=hmm_folder, threshold=threshold) + raise RuntimeError('Invalid Pocketsphinx options: ' + str(options)) + + def _transcribe(self, byte_data): + self.decoder.start_utt() + self.decoder.process_raw(byte_data, False, False) + self.decoder.end_utt() + return self.decoder.hyp() + + def found_wake_word(self, frame_data): + hyp = self._transcribe(frame_data + b'\0' * int(2 * 16000 * 0.01)) + return bool(hyp and self.key_phrase in hyp.hypstr.lower()) + + def update(self, stream: Union[BinaryIO, np.ndarray, bytes]) -> float: + if isinstance(stream, np.ndarray): + chunk = audio_to_buffer(stream) + else: + if isinstance(stream, (bytes, bytearray)): + chunk = stream + else: + chunk = stream.read(self.read_size) + if len(chunk) == 0: + raise EOFError + self.buffer = self.buffer[len(chunk):] + chunk + return float(self.found_wake_word(self.buffer)) diff --git a/precise/pocketsphinx/scripts/__init__.py b/precise/pocketsphinx/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/precise/pocketsphinx/scripts/listen.py b/precise/pocketsphinx/scripts/listen.py new file mode 100755 index 00000000..0403fc86 --- /dev/null +++ b/precise/pocketsphinx/scripts/listen.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# Copyright 2019 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from precise_runner import PreciseRunner +from precise_runner.runner import ListenerEngine +from prettyparse import Usage +from threading import Event + +from precise.pocketsphinx.listener import PocketsphinxListener +from precise.scripts.base_script import BaseScript +from precise.util import activate_notify + + +class PocketsphinxListenScript(BaseScript): + usage = Usage(''' + Run Pocketsphinx on microphone audio input + + :key_phrase str + Key phrase composed of words from dictionary + + :dict_file str + Filename of dictionary with word pronunciations + + :hmm_folder str + Folder containing hidden markov model + + :-th --threshold str 1e-90 + Threshold for activations + + :-c --chunk-size int 2048 + Samples between inferences + ''') + + def run(self): + def on_activation(): + activate_notify() + + def on_prediction(conf): + print('!' if conf > 0.5 else '.', end='', flush=True) + + args = self.args + runner = PreciseRunner( + ListenerEngine( + PocketsphinxListener( + args.key_phrase, args.dict_file, args.hmm_folder, args.threshold, args.chunk_size + ) + ), 3, on_activation=on_activation, on_prediction=on_prediction + ) + runner.start() + Event().wait() # Wait forever + + +main = PocketsphinxListenScript.run_main + +if __name__ == '__main__': + main() diff --git a/precise/pocketsphinx/scripts/test.py b/precise/pocketsphinx/scripts/test.py new file mode 100755 index 00000000..c782ce83 --- /dev/null +++ b/precise/pocketsphinx/scripts/test.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# Copyright 2019 Mycroft AI Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import wave +from prettyparse import Usage +from subprocess import check_output, PIPE + +from precise.pocketsphinx.listener import PocketsphinxListener +from precise.scripts.base_script import BaseScript +from precise.scripts.test import Stats +from precise.train_data import TrainData + + +class PocketsphinxTestScript(BaseScript): + usage = Usage(''' + Test a dataset using Pocketsphinx + + :key_phrase str + Key phrase composed of words from dictionary + + :dict_file str + Filename of dictionary with word pronunciations + + :hmm_folder str + Folder containing hidden markov model + + :-th --threshold str 1e-90 + Threshold for activations + + :-t --use-train + Evaluate training data instead of test data + + :-nf --no-filenames + Don't show the names of files that failed + + ... + ''') | TrainData.usage + + def __init__(self, args): + super().__init__(args) + self.listener = PocketsphinxListener( + args.key_phrase, args.dict_file, args.hmm_folder, args.threshold + ) + + self.outputs = [] + self.targets = [] + self.filenames = [] + + def get_stats(self): + return Stats(self.outputs, self.targets, self.filenames) + + def run(self): + args = self.args + data = TrainData.from_both(args.tags_file, args.tags_folder, args.folder) + print('Data:', data) + + ww_files, nww_files = data.train_files if args.use_train else data.test_files + self.run_test(ww_files, 'Wake Word', 1.0) + self.run_test(nww_files, 'Not Wake Word', 0.0) + stats = self.get_stats() + if not self.args.no_filenames: + fp_files = stats.calc_filenames(False, True, 0.5) + fn_files = stats.calc_filenames(False, False, 0.5) + print('=== False Positives ===') + print('\n'.join(fp_files)) + print() + print('=== False Negatives ===') + print('\n'.join(fn_files)) + print() + print(stats.counts_str(0.5)) + print() + print(stats.summary_str(0.5)) + + def eval_file(self, filename) -> float: + transcription = check_output( + ['pocketsphinx_continuous', '-kws_threshold', '1e-20', '-keyphrase', 'hey my craft', + '-infile', filename], stderr=PIPE) + return float(bool(transcription) and not transcription.isspace()) + + def run_test(self, test_files, label_name, label): + print() + print('===', label_name, '===') + for test_file in test_files: + try: + with wave.open(test_file) as wf: + frames = wf.readframes(wf.getnframes()) + except (OSError, EOFError): + print('?', end='', flush=True) + continue + + out = int(self.listener.found_wake_word(frames)) + self.outputs.append(out) + self.targets.append(label) + self.filenames.append(test_file) + print('!' if out else '.', end='', flush=True) + print() + + +main = PocketsphinxTestScript.run_main + +if __name__ == '__main__': + main() From 149a649f422979a676d3405f59415d2a6e33a14b Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Tue, 21 Apr 2020 09:00:14 -0500 Subject: [PATCH 13/18] Adds quick fix to allow save as h5. --- precise/scripts/train_incremental.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/precise/scripts/train_incremental.py b/precise/scripts/train_incremental.py index ef2082d7..408ee51c 100644 --- a/precise/scripts/train_incremental.py +++ b/precise/scripts/train_incremental.py @@ -107,7 +107,8 @@ def retrain(self): validation_data=test_data, callbacks=self.callbacks, initial_epoch=self.epoch ) finally: - self.listener.runner.model.save(self.args.model,save_format='h5') + self.listener.runner.model.save(self.args.model + '.h5') # Save with '.h5' file extension to force format + os.rename(self.args.model + '.h5', self.args.model) # Rename with original def train_on_audio(self, fn: str): """Run through a single audio file""" From 2b1bb37049a9bcad76457dfa48b7e18e7db60bc9 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Tue, 21 Apr 2020 09:02:03 -0500 Subject: [PATCH 14/18] Bugfix: Imports rename from os. --- precise/scripts/train_incremental.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/precise/scripts/train_incremental.py b/precise/scripts/train_incremental.py index 408ee51c..67d744ee 100644 --- a/precise/scripts/train_incremental.py +++ b/precise/scripts/train_incremental.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import numpy as np -from os import makedirs +from os import makedirs, rename from os.path import basename, splitext, isfile, join from prettyparse import Usage from random import random @@ -108,7 +108,7 @@ def retrain(self): ) finally: self.listener.runner.model.save(self.args.model + '.h5') # Save with '.h5' file extension to force format - os.rename(self.args.model + '.h5', self.args.model) # Rename with original + rename(self.args.model + '.h5', self.args.model) # Rename with original def train_on_audio(self, fn: str): """Run through a single audio file""" From e2a3c2dc58678371281b2b8b5c42a9844b6e4d90 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Wed, 29 Apr 2020 10:58:44 -0500 Subject: [PATCH 15/18] Fixes saving model to .h5 in train_generated.py --- precise/scripts/train_generated.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/precise/scripts/train_generated.py b/precise/scripts/train_generated.py index 715c1cf8..2f0fa50f 100644 --- a/precise/scripts/train_generated.py +++ b/precise/scripts/train_generated.py @@ -233,7 +233,8 @@ def run(self): callbacks=self.callbacks, initial_epoch=self.epoch ) finally: - self.model.save(self.args.model, save_format='h5') + self.listener.runner.model.save(self.args.model + '.h5') # Save with '.h5' file extension to force format + rename(self.args.model + '.h5', self.args.model) # Rename with original save_params(self.args.model) From 3baf5da2489130bd300c6859da13be7f60db8788 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Wed, 8 Jul 2020 12:04:50 -0500 Subject: [PATCH 16/18] Allows for generator training. --- precise/scripts/train_generated.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/precise/scripts/train_generated.py b/precise/scripts/train_generated.py index 2f0fa50f..e9c52502 100644 --- a/precise/scripts/train_generated.py +++ b/precise/scripts/train_generated.py @@ -19,6 +19,7 @@ from contextlib import suppress from fitipy import Fitipy from keras.callbacks import LambdaCallback +from os import rename from os.path import splitext, join, basename from prettyparse import Usage from random import random, shuffle @@ -91,7 +92,7 @@ def __init__(self, args): self.listener = Listener('', args.chunk_size, runner_cls=lambda x: None) from keras.callbacks import ModelCheckpoint - checkpoint = ModelCheckpoint(args.model, monitor=args.metric_monitor, + checkpoint = ModelCheckpoint(args.model + '.pb', monitor=args.metric_monitor, save_best_only=args.save_best) epoch_fiti = Fitipy(splitext(args.model)[0] + '.epoch') self.epoch = epoch_fiti.read().read(0, int) @@ -224,16 +225,16 @@ def generate_samples(self): def run(self): """Train the model on randomly generated batches""" - _, test_data = self.data.load(train=False, test=True) + _, test_data = self.data.load(train=True, test=True) try: - self.model.fit_generator( + self.model.fit( self.samples_to_batches(self.generate_samples(), self.args.batch_size), steps_per_epoch=self.args.steps_per_epoch, epochs=self.epoch + self.args.epochs, validation_data=test_data, callbacks=self.callbacks, initial_epoch=self.epoch ) finally: - self.listener.runner.model.save(self.args.model + '.h5') # Save with '.h5' file extension to force format + self.model.save(self.args.model + '.h5') # Save with '.h5' file extension to force format rename(self.args.model + '.h5', self.args.model) # Rename with original save_params(self.args.model) From 9e7d8841b0e459253f0ed184e940290537b6724b Mon Sep 17 00:00:00 2001 From: "Matthew D. Scholefield" Date: Fri, 17 Jul 2020 00:17:57 -0500 Subject: [PATCH 17/18] Swap out keras for tf.keras Also upgrades tensorflow to 2.2.0 --- precise/functions.py | 8 ++++---- precise/model.py | 11 +++++------ precise/network_runner.py | 8 +------- precise/scripts/train.py | 4 ++-- precise/scripts/train_generated.py | 4 ++-- setup.py | 3 +-- 6 files changed, 15 insertions(+), 23 deletions(-) diff --git a/precise/functions.py b/precise/functions.py index be59596a..7bcf69a8 100644 --- a/precise/functions.py +++ b/precise/functions.py @@ -33,7 +33,7 @@ def weighted_log_loss(yt, yp) -> Any: yt: Target yp: Prediction """ - from keras import backend as K + import tensorflow.keras.backend as K pos_loss = -(0 + yt) * K.log(0 + yp + K.epsilon()) neg_loss = -(1 - yt) * K.log(1 - yp + K.epsilon()) @@ -42,7 +42,7 @@ def weighted_log_loss(yt, yp) -> Any: def weighted_mse_loss(yt, yp) -> Any: - from keras import backend as K + import tensorflow.keras.backend as K total = K.sum(K.ones_like(yt)) neg_loss = total * K.sum(K.square(yp * (1 - yt))) / K.sum(1 - yt) @@ -52,12 +52,12 @@ def weighted_mse_loss(yt, yp) -> Any: def false_pos(yt, yp) -> Any: - from keras import backend as K + import tensorflow.keras.backend as K return K.sum(K.cast(yp * (1 - yt) > 0.5, 'float')) / K.maximum(1.0, K.sum(1 - yt)) def false_neg(yt, yp) -> Any: - from keras import backend as K + import tensorflow.keras.backend as K return K.sum(K.cast((1 - yp) * (0 + yt) > 0.5, 'float')) / K.maximum(1.0, K.sum(0 + yt)) diff --git a/precise/model.py b/precise/model.py index 6ee90ca6..63a1df74 100644 --- a/precise/model.py +++ b/precise/model.py @@ -19,7 +19,7 @@ from precise.params import inject_params, pr if TYPE_CHECKING: - from keras.models import Sequential + from tensorflow.keras.models import Sequential @attr.s() @@ -45,7 +45,8 @@ def load_precise_model(model_name: str) -> Any: print('Warning: Unknown model type, ', model_name) inject_params(model_name) - return load_keras().models.load_model(model_name) + from tensorflow.keras.models import load_model + return load_model(model_name, custom_objects=globals()) def create_model(model_name: Optional[str], params: ModelParams) -> 'Sequential': @@ -63,9 +64,8 @@ def create_model(model_name: Optional[str], params: ModelParams) -> 'Sequential' print('Loading from ' + model_name + '...') model = load_precise_model(model_name) else: - from keras.layers.core import Dense - from keras.layers.recurrent import GRU - from keras.models import Sequential + from tensorflow.keras.layers import Dense, GRU + from tensorflow.keras.models import Sequential model = Sequential() model.add(GRU( @@ -74,7 +74,6 @@ def create_model(model_name: Optional[str], params: ModelParams) -> 'Sequential' )) model.add(Dense(1, activation='sigmoid')) - load_keras() metrics = ['accuracy'] + params.extra_metrics * [false_pos, false_neg] set_loss_bias(params.loss_bias) for i in model.layers[:params.freeze_till]: diff --git a/precise/network_runner.py b/precise/network_runner.py index 78907076..5fe7b624 100644 --- a/precise/network_runner.py +++ b/precise/network_runner.py @@ -68,21 +68,15 @@ def run(self, inp: np.ndarray) -> float: class KerasRunner(Runner): def __init__(self, model_name: str): - # Load model using Keras (not tf.keras) self.model = load_precise_model(model_name) - - # TF 2.0 doesn't work well with sessions and graphs - # Only in tf.v1.compat, but that restricts usage of v2 features - self.graph = None def predict(self, inputs: np.ndarray): - import keras as K - K.backend.tensorflow_backend._SYMBOLIC_SCOPE.value = True return self.model.predict(inputs) def run(self, inp: np.ndarray) -> float: return self.predict(inp[np.newaxis])[0][0] + class TFLiteRunner(Runner): def __init__(self, model_name: str): import tensorflow as tf diff --git a/precise/scripts/train.py b/precise/scripts/train.py index bbd2aa3d..7b59626c 100644 --- a/precise/scripts/train.py +++ b/precise/scripts/train.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from fitipy import Fitipy -from keras.callbacks import LambdaCallback +from tensorflow.keras.callbacks import LambdaCallback from os.path import splitext, isfile from prettyparse import Usage from typing import Any, Tuple @@ -85,7 +85,7 @@ def __init__(self, args): self.model = create_model(args.model, params) self.train, self.test = self.load_data(self.args) - from keras.callbacks import ModelCheckpoint + from tensorflow.keras.callbacks import ModelCheckpoint checkpoint = ModelCheckpoint(args.model, monitor=args.metric_monitor, save_best_only=args.save_best) epoch_fiti = Fitipy(splitext(args.model)[0] + '.epoch') diff --git a/precise/scripts/train_generated.py b/precise/scripts/train_generated.py index 715c1cf8..d5453f9a 100644 --- a/precise/scripts/train_generated.py +++ b/precise/scripts/train_generated.py @@ -18,7 +18,7 @@ import numpy as np from contextlib import suppress from fitipy import Fitipy -from keras.callbacks import LambdaCallback +from tensorflow.keras.callbacks import LambdaCallback from os.path import splitext, join, basename from prettyparse import Usage from random import random, shuffle @@ -90,7 +90,7 @@ def __init__(self, args): self.model = create_model(args.model, params) self.listener = Listener('', args.chunk_size, runner_cls=lambda x: None) - from keras.callbacks import ModelCheckpoint + from tensorflow.keras.callbacks import ModelCheckpoint checkpoint = ModelCheckpoint(args.model, monitor=args.metric_monitor, save_best_only=args.save_best) epoch_fiti = Fitipy(splitext(args.model)[0] + '.epoch') diff --git a/setup.py b/setup.py index 9035431e..cb3db311 100644 --- a/setup.py +++ b/setup.py @@ -69,10 +69,9 @@ }, install_requires=[ 'numpy', - 'tensorflow-gpu==2.2.0rc', # This should be changed for 2.2.0 when it's released. + 'tensorflow-gpu==2.2.0', 'sonopy', 'pyaudio', - 'keras>2.1.5', 'h5py', 'wavio', 'typing', From 0e0ac5b8b14ff6a6ecffd300c40049131990e8c9 Mon Sep 17 00:00:00 2001 From: Andres Elizondo Date: Wed, 19 Aug 2020 14:37:05 -0500 Subject: [PATCH 18/18] Restores pocketsphinx. --- setup.py | 4 +++- setup.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index cb3db311..44b22d98 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,9 @@ ], packages=[ 'precise', - 'precise.scripts' + 'precise.scripts', + 'precise.pocketsphinx', + 'precise.pocketsphinx.scripts' ], entry_points={ 'console_scripts': [ diff --git a/setup.sh b/setup.sh index c72c1b87..1c8a6a0b 100755 --- a/setup.sh +++ b/setup.sh @@ -60,4 +60,4 @@ fi pip install -e runner/ pip install -e . -#pip install pocketsphinx # Optional, for comparison +pip install pocketsphinx # Optional, for comparison