mycroft_talknet.py

import os
from flask import Flask, request, send_file
import sys
import torch
import numpy as np
from scipy.io import wavfile
import io
from nemo.collections.tts.models import TalkNetSpectModel
from nemo.collections.tts.models import TalkNetPitchModel
from nemo.collections.tts.models import TalkNetDursModel
import json

sys.path.append("hifi-gan")
from env import AttrDict
from meldataset import MAX_WAV_VALUE
from models import Generator
from denoiser import Denoiser


app = Flask(__name__)
RUN_PATH = os.path.dirname(os.path.realpath(__file__))
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"


def load_hifigan(model_name, conf_name):
    # Load HiFi-GAN
    conf = os.path.join("hifi-gan", conf_name + ".json")
    with open(conf) as f:
        json_config = json.loads(f.read())
    h = AttrDict(json_config)
    torch.manual_seed(h.seed)
    hifigan = Generator(h).to(torch.device(DEVICE))
    state_dict_g = torch.load(model_name, map_location=torch.device(DEVICE))
    hifigan.load_state_dict(state_dict_g["generator"])
    hifigan.eval()
    hifigan.remove_weight_norm()
    denoiser = Denoiser(hifigan, mode="normal")
    return hifigan, h, denoiser


def generate_json(input, outpath):
    output = ""
    sample_rate = 22050
    lpath = input.split("|")[0].strip()
    size = os.stat(lpath).st_size
    x = {
        "audio_filepath": lpath,
        "duration": size / (sample_rate * 2),
        "text": input.split("|")[1].strip(),
    }
    output += json.dumps(x) + "\n"
    with open(outpath, "w", encoding="utf8") as w:
        w.write(output)


def load_talknet(talknet_path):
    with torch.no_grad():
        tnmodel = TalkNetSpectModel.restore_from(talknet_path)
        durs_path = os.path.join(os.path.dirname(talknet_path), "TalkNetDurs.nemo")
        tndurs = TalkNetDursModel.restore_from(durs_path)
        tnmodel.add_module("_durs_model", tndurs)
        pitch_path = os.path.join(os.path.dirname(talknet_path), "TalkNetPitch.nemo")
        tnpitch = TalkNetPitchModel.restore_from(pitch_path)
        tnmodel.add_module("_pitch_model", tnpitch)
        tnmodel.eval()
        return tnmodel


def generate_audio(transcript, tnmodel, hifigan, denoiser):
    with torch.no_grad():
        tokens = tnmodel.parse(text=transcript.strip())
        spect = tnmodel.generate_spectrogram(tokens=tokens)

        y_g_hat = hifigan(spect.float())
        audio = y_g_hat.squeeze()
        audio = audio * MAX_WAV_VALUE
        audio_denoised = denoiser(audio.view(1, -1), strength=35)[:, 0]
        audio_np = audio_denoised.detach().cpu().numpy().reshape(-1).astype(np.int16)

        buffer = io.BytesIO()
        wavfile.write(buffer, 22050, audio_np)
        return buffer


@app.route("/", methods=["GET"])
def get_check():
    return "TalkNet server online"


@app.route("/api/tts", methods=["GET"])
def get_tts():
    if "text" not in request.args:
        return ""
    transcript = request.args.get("text")
    return send_file(
        generate_audio(transcript, tnmodel, hifigan, denoiser),
        attachment_filename="audio.wav",
        mimetype="audio/x-wav",
    )


if __name__ == "__main__":
    hifigan, h, denoiser = load_hifigan(
        os.path.join(
            RUN_PATH, "models", "1QnOliOAmerMUNuo2wXoH-YoainoSjZen", "hifiganmodel"
        ),
        "config_v1",
    )
    tnmodel = load_talknet(
        os.path.join(
            RUN_PATH, "models", "1QnOliOAmerMUNuo2wXoH-YoainoSjZen", "TalkNetSpect.nemo"
        )
    )
    app.run(debug=False)