07_02_lstm_compose_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Compose: Training a model to generate music"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import numpy\n",
    "from music21 import note, chord\n",
    "\n",
    "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
    "from keras.utils import plot_model\n",
    "\n",
    "from models.RNNAttention import get_distinct, create_lookups, prepare_sequences, get_music_list, create_network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# run params\n",
    "section = 'compose'\n",
    "run_id = '0010'\n",
    "music_name = 'cello'\n",
    "\n",
    "run_folder = 'run/{}/'.format(section)\n",
    "run_folder += '_'.join([run_id, music_name])\n",
    "\n",
    "\n",
    "store_folder = os.path.join(run_folder, 'store')\n",
    "data_folder = os.path.join('bach', music_name)\n",
    "\n",
    "if not os.path.exists(run_folder):\n",
    "    os.makedirs(run_folder)\n",
    "    os.mkdir(os.path.join(run_folder, 'store'))\n",
    "    os.mkdir(os.path.join(run_folder, 'output'))\n",
    "    os.mkdir(os.path.join(run_folder, 'weights'))\n",
    "    os.mkdir(os.path.join(run_folder, 'viz'))\n",
    "    \n",
    "\n",
    "\n",
    "mode = 'build' # 'load' # \n",
    "\n",
    "# data params\n",
    "intervals = range(-2,3)\n",
    "seq_len = 32\n",
    "\n",
    "# model params\n",
    "embed_size = 100\n",
    "rnn_units = 128\n",
    "use_attention = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extract the notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "36 files in total\n",
      "1 Parsing bach/cello/cs3-2all.mid\n",
      "2 Parsing bach/cello/cs2-6gig.mid\n",
      "3 Parsing bach/cello/cs6-3cou.mid\n",
      "4 Parsing bach/cello/cs5-5gav.mid\n",
      "5 Parsing bach/cello/cs2-4sar.mid\n",
      "6 Parsing bach/cello/cs2-5men.mid\n",
      "7 Parsing bach/cello/cs4-2all.mid\n",
      "8 Parsing bach/cello/cs5-4sar.mid\n",
      "9 Parsing bach/cello/cs5-6gig.mid\n",
      "10 Parsing bach/cello/cs1-3cou.mid\n",
      "11 Parsing bach/cello/cs1-1pre.mid\n",
      "12 Parsing bach/cello/cs3-6gig.mid\n",
      "13 Parsing bach/cello/cs3-4sar.mid\n",
      "14 Parsing bach/cello/cs2-2all.mid\n",
      "15 Parsing bach/cello/cs4-4sar.mid\n",
      "16 Parsing bach/cello/cs6-1pre.mid\n",
      "17 Parsing bach/cello/cs4-6gig.mid\n",
      "18 Parsing bach/cello/cs5-2all.mid\n",
      "19 Parsing bach/cello/cs6-2all.mid\n",
      "20 Parsing bach/cello/cs3-3cou.mid\n",
      "21 Parsing bach/cello/cs5-1pre.mid\n",
      "22 Parsing bach/cello/cs1-2all.mid\n",
      "23 Parsing bach/cello/cs2-1pre.mid\n",
      "24 Parsing bach/cello/cs4-3cou.mid\n",
      "25 Parsing bach/cello/cs6-4sar.mid\n",
      "26 Parsing bach/cello/cs6-6gig.mid\n",
      "27 Parsing bach/cello/cs4-1pre.mid\n",
      "28 Parsing bach/cello/cs2-3cou.mid\n",
      "29 Parsing bach/cello/cs3-5bou.mid\n",
      "30 Parsing bach/cello/cs1-6gig.mid\n",
      "31 Parsing bach/cello/cs5-3cou.mid\n",
      "32 Parsing bach/cello/cs3-1pre.mid\n",
      "33 Parsing bach/cello/cs1-5men.mid\n",
      "34 Parsing bach/cello/cs6-5gav.mid\n",
      "35 Parsing bach/cello/cs1-4sar.mid\n",
      "36 Parsing bach/cello/cs4-5bou.mid\n"
     ]
    }
   ],
   "source": [
    "if mode == 'build':\n",
    "    \n",
    "    music_list, parser = get_music_list(data_folder)\n",
    "    print(len(music_list), 'files in total')\n",
    "\n",
    "    notes = []\n",
    "    durations = []\n",
    "\n",
    "    for i, file in enumerate(music_list):\n",
    "        print(i+1, \"Parsing %s\" % file)\n",
    "        original_score = parser.parse(file).chordify()\n",
    "        \n",
    "\n",
    "        for interval in intervals:\n",
    "\n",
    "            score = original_score.transpose(interval)\n",
    "\n",
    "            notes.extend(['START'] * seq_len)\n",
    "            durations.extend([0]* seq_len)\n",
    "\n",
    "            for element in score.flatten():\n",
    "                \n",
    "                if isinstance(element, note.Note):\n",
    "                    if element.isRest:\n",
    "                        notes.append(str(element.name))\n",
    "                        durations.append(element.duration.quarterLength)\n",
    "                    else:\n",
    "                        notes.append(str(element.nameWithOctave))\n",
    "                        durations.append(element.duration.quarterLength)\n",
    "\n",
    "                if isinstance(element, chord.Chord):\n",
    "                    notes.append('.'.join(n.nameWithOctave for n in element.pitches))\n",
    "                    durations.append(element.duration.quarterLength)\n",
    "\n",
    "    with open(os.path.join(store_folder, 'notes'), 'wb') as f:\n",
    "        pickle.dump(notes, f) #['G2', 'D3', 'B3', 'A3', 'B3', 'D3', 'B3', 'D3', 'G2',...]\n",
    "    with open(os.path.join(store_folder, 'durations'), 'wb') as f:\n",
    "        pickle.dump(durations, f) \n",
    "else:\n",
    "    with open(os.path.join(store_folder, 'notes'), 'rb') as f:\n",
    "        notes = pickle.load(f) #['G2', 'D3', 'B3', 'A3', 'B3', 'D3', 'B3', 'D3', 'G2',...]\n",
    "    with open(os.path.join(store_folder, 'durations'), 'rb') as f:\n",
    "        durations = pickle.load(f) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the lookup tables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# get the distinct sets of notes and durations\n",
    "note_names, n_notes = get_distinct(notes)\n",
    "duration_names, n_durations = get_distinct(durations)\n",
    "distincts = [note_names, n_notes, duration_names, n_durations]\n",
    "\n",
    "with open(os.path.join(store_folder, 'distincts'), 'wb') as f:\n",
    "    pickle.dump(distincts, f)\n",
    "\n",
    "# make the lookup dictionaries for notes and dictionaries and save\n",
    "note_to_int, int_to_note = create_lookups(note_names)\n",
    "duration_to_int, int_to_duration = create_lookups(duration_names)\n",
    "lookups = [note_to_int, int_to_note, duration_to_int, int_to_duration]\n",
    "\n",
    "with open(os.path.join(store_folder, 'lookups'), 'wb') as f:\n",
    "    pickle.dump(lookups, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "note_to_int\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'A2': 0,\n",
       " 'A2.A3': 1,\n",
       " 'A2.A3.C#4.E-4': 2,\n",
       " 'A2.A3.C#4.F#4': 3,\n",
       " 'A2.A3.F#4': 4,\n",
       " 'A2.B-2': 5,\n",
       " 'A2.B-2.C3': 6,\n",
       " 'A2.B-2.C3.D3': 7,\n",
       " 'A2.B-3': 8,\n",
       " 'A2.B2': 9,\n",
       " 'A2.B3': 10,\n",
       " 'A2.C#3': 11,\n",
       " 'A2.C#3.D3.G3': 12,\n",
       " 'A2.C#3.G#3': 13,\n",
       " 'A2.C#3.G3': 14,\n",
       " 'A2.C#4': 15,\n",
       " 'A2.C#4.A4': 16,\n",
       " 'A2.C#4.E4': 17,\n",
       " 'A2.C3': 18,\n",
       " 'A2.C3.A3.E4': 19,\n",
       " 'A2.C4': 20,\n",
       " 'A2.D3': 21,\n",
       " 'A2.D3.C4': 22,\n",
       " 'A2.D3.D4': 23,\n",
       " 'A2.D4': 24,\n",
       " 'A2.E-3': 25,\n",
       " 'A2.E-3.C#4': 26,\n",
       " 'A2.E-3.C4': 27,\n",
       " 'A2.E3': 28,\n",
       " 'A2.E3.A3': 29,\n",
       " 'A2.E3.B-3': 30,\n",
       " 'A2.E3.B3': 31,\n",
       " 'A2.E3.B3.A4': 32,\n",
       " 'A2.E3.C#4': 33,\n",
       " 'A2.E3.C#4.A4': 34,\n",
       " 'A2.E3.C#4.B4': 35,\n",
       " 'A2.E3.C#4.E4': 36,\n",
       " 'A2.E3.C#4.G#4': 37,\n",
       " 'A2.E3.C#4.G4': 38,\n",
       " 'A2.E3.C4': 39,\n",
       " 'A2.E3.D4': 40,\n",
       " 'A2.E3.D4.A4': 41,\n",
       " 'A2.E3.E-4': 42,\n",
       " 'A2.E3.E4': 43,\n",
       " 'A2.E3.G3.A3.B-3': 44,\n",
       " 'A2.F#3': 45,\n",
       " 'A2.F#3.C#4': 46,\n",
       " 'A2.F#3.C4': 47,\n",
       " 'A2.F#3.D4': 48,\n",
       " 'A2.F#3.D4.A4': 49,\n",
       " 'A2.F#3.D4.E4': 50,\n",
       " 'A2.F#3.D4.F#4': 51,\n",
       " 'A2.F#3.E-4': 52,\n",
       " 'A2.F#3.E-4.B4': 53,\n",
       " 'A2.F#3.E4': 54,\n",
       " 'A2.F#3.G3': 55,\n",
       " 'A2.F#4': 56,\n",
       " 'A2.F3': 57,\n",
       " 'A2.F3.C4': 58,\n",
       " 'A2.F3.C4.E-4': 59,\n",
       " 'A2.F3.D4': 60,\n",
       " 'A2.F3.D4.A4': 61,\n",
       " 'A2.F3.E-4': 62,\n",
       " 'A2.F3.E-4.C5': 63,\n",
       " 'A2.G#3': 64,\n",
       " 'A2.G#3.E4': 65,\n",
       " 'A2.G#3.F4': 66,\n",
       " 'A2.G3': 67,\n",
       " 'A2.G3.A3': 68,\n",
       " 'A2.G3.C#4': 69,\n",
       " 'A2.G3.C4': 70,\n",
       " 'A2.G3.D4': 71,\n",
       " 'A2.G3.E4': 72,\n",
       " 'A2.G4': 73,\n",
       " 'A3': 74,\n",
       " 'A3.B-3': 75,\n",
       " 'A3.B-3.D4': 76,\n",
       " 'A3.B-3.E-4': 77,\n",
       " 'A3.B3': 78,\n",
       " 'A3.B3.C#4': 79,\n",
       " 'A3.B3.C#4.D4': 80,\n",
       " 'A3.B3.C4': 81,\n",
       " 'A3.B3.C4.D4': 82,\n",
       " 'A3.C#4': 83,\n",
       " 'A3.C#4.C#5': 84,\n",
       " 'A3.C#4.E4': 85,\n",
       " 'A3.C#4.F#4': 86,\n",
       " 'A3.C#4.G#4': 87,\n",
       " 'A3.C4': 88,\n",
       " 'A3.D4': 89,\n",
       " 'A3.E-4': 90,\n",
       " 'A3.E-4.F4': 91,\n",
       " 'A3.E4': 92,\n",
       " 'A3.E4.C#5': 93,\n",
       " 'A3.E4.F#4': 94,\n",
       " 'A3.E4.F4': 95,\n",
       " 'A3.E4.F4.G4': 96,\n",
       " 'A3.E4.G4': 97,\n",
       " 'A3.F#4': 98,\n",
       " 'A3.F#4.G4': 99,\n",
       " 'A3.F4': 100,\n",
       " 'A3.G4': 101,\n",
       " 'A4': 102,\n",
       " 'A4.B-4': 103,\n",
       " 'A4.B-4.C5.C#5': 104,\n",
       " 'A4.B4': 105,\n",
       " 'A4.C5': 106,\n",
       " 'A5': 107,\n",
       " 'B-1': 108,\n",
       " 'B-1.A2.E-3.F#3': 109,\n",
       " 'B-1.B-2': 110,\n",
       " 'B-1.C#3': 111,\n",
       " 'B-1.C#3.F3': 112,\n",
       " 'B-1.D3.F3': 113,\n",
       " 'B-1.F#2': 114,\n",
       " 'B-1.F#2.C3.A3': 115,\n",
       " 'B-1.F2': 116,\n",
       " 'B-1.F2.C#3': 117,\n",
       " 'B-1.F2.C#3.B-3': 118,\n",
       " 'B-1.F2.C#3.G#3': 119,\n",
       " 'B-1.F2.C3': 120,\n",
       " 'B-1.F2.D3': 121,\n",
       " 'B-1.F2.D3.B-3': 122,\n",
       " 'B-1.F2.D3.G#3': 123,\n",
       " 'B-1.F2.E-3': 124,\n",
       " 'B-1.F2.E-3.B-3': 125,\n",
       " 'B-1.G#2': 126,\n",
       " 'B-1.G#2.D3': 127,\n",
       " 'B-1.G2.C#3': 128,\n",
       " 'B-1.G2.E-3.G3': 129,\n",
       " 'B-1.G2.E3': 130,\n",
       " 'B-1.G2.E3.C4': 131,\n",
       " 'B-1.G2.F3.G3': 132,\n",
       " 'B-2': 133,\n",
       " 'B-2.A3': 134,\n",
       " 'B-2.B-3': 135,\n",
       " 'B-2.B2': 136,\n",
       " 'B-2.B2.C#3': 137,\n",
       " 'B-2.B3': 138,\n",
       " 'B-2.C#3': 139,\n",
       " 'B-2.C#3.B-3.F4': 140,\n",
       " 'B-2.C3': 141,\n",
       " 'B-2.C3.D3': 142,\n",
       " 'B-2.D3': 143,\n",
       " 'B-2.D3.A3': 144,\n",
       " 'B-2.D3.D4': 145,\n",
       " 'B-2.D3.E-3.G#3': 146,\n",
       " 'B-2.D3.E4': 147,\n",
       " 'B-2.D3.G#3': 148,\n",
       " 'B-2.D3.G3': 149,\n",
       " 'B-2.E-3': 150,\n",
       " 'B-2.E-3.C#4': 151,\n",
       " 'B-2.E-3.E-4': 152,\n",
       " 'B-2.E3': 153,\n",
       " 'B-2.E3.C#4': 154,\n",
       " 'B-2.E3.D4': 155,\n",
       " 'B-2.F#3': 156,\n",
       " 'B-2.F#3.C#4': 157,\n",
       " 'B-2.F#3.C#4.E4': 158,\n",
       " 'B-2.F#3.E-4': 159,\n",
       " 'B-2.F#3.E-4.B-4': 160,\n",
       " 'B-2.F#3.E4': 161,\n",
       " 'B-2.F#3.E4.C#5': 162,\n",
       " 'B-2.F3': 163,\n",
       " 'B-2.F3.B-3': 164,\n",
       " 'B-2.F3.B3': 165,\n",
       " 'B-2.F3.C#4': 166,\n",
       " 'B-2.F3.C4': 167,\n",
       " 'B-2.F3.D4': 168,\n",
       " 'B-2.F3.D4.A4': 169,\n",
       " 'B-2.F3.D4.B-4': 170,\n",
       " 'B-2.F3.D4.F4': 171,\n",
       " 'B-2.F3.D4.G#4': 172,\n",
       " 'B-2.F3.E-4': 173,\n",
       " 'B-2.F3.G#3': 174,\n",
       " 'B-2.F3.G#3.B-3.B3': 175,\n",
       " 'B-2.F3.G3': 176,\n",
       " 'B-2.G#3': 177,\n",
       " 'B-2.G#3.D4': 178,\n",
       " 'B-2.G#3.E-4': 179,\n",
       " 'B-2.G#4': 180,\n",
       " 'B-2.G3': 181,\n",
       " 'B-2.G3.C#4': 182,\n",
       " 'B-2.G3.D4': 183,\n",
       " 'B-2.G3.E-4': 184,\n",
       " 'B-2.G3.E-4.B-4': 185,\n",
       " 'B-2.G3.E-4.F4': 186,\n",
       " 'B-2.G3.E-4.G4': 187,\n",
       " 'B-2.G3.E4': 188,\n",
       " 'B-2.G3.F4': 189,\n",
       " 'B-2.G4': 190,\n",
       " 'B-3': 191,\n",
       " 'B-3.B3': 192,\n",
       " 'B-3.B3.E-4': 193,\n",
       " 'B-3.B3.E4': 194,\n",
       " 'B-3.C#4': 195,\n",
       " 'B-3.C4': 196,\n",
       " 'B-3.C4.C#4': 197,\n",
       " 'B-3.C4.C#4.E-4': 198,\n",
       " 'B-3.C4.D4': 199,\n",
       " 'B-3.C4.D4.E-4': 200,\n",
       " 'B-3.C4.D4.E4': 201,\n",
       " 'B-3.D4': 202,\n",
       " 'B-3.D4.F4': 203,\n",
       " 'B-3.E-4': 204,\n",
       " 'B-3.E4': 205,\n",
       " 'B-3.E4.F#4': 206,\n",
       " 'B-3.F#4': 207,\n",
       " 'B-3.F4': 208,\n",
       " 'B-3.F4.F#4': 209,\n",
       " 'B-3.F4.F#4.G#4': 210,\n",
       " 'B-3.F4.G#4': 211,\n",
       " 'B-3.F4.G4': 212,\n",
       " 'B-3.G#4': 213,\n",
       " 'B-3.G4': 214,\n",
       " 'B-3.G4.G#4': 215,\n",
       " 'B-4': 216,\n",
       " 'B-4.B4': 217,\n",
       " 'B-4.B4.C#5.D5': 218,\n",
       " 'B-4.C#5': 219,\n",
       " 'B-4.C5': 220,\n",
       " 'B1': 221,\n",
       " 'B1.A2': 222,\n",
       " 'B1.A2.E-3': 223,\n",
       " 'B1.B-2.E3.G3': 224,\n",
       " 'B1.B2': 225,\n",
       " 'B1.D3': 226,\n",
       " 'B1.D3.F#3': 227,\n",
       " 'B1.E-3.F#3': 228,\n",
       " 'B1.F#2': 229,\n",
       " 'B1.F#2.C#3': 230,\n",
       " 'B1.F#2.D3': 231,\n",
       " 'B1.F#2.D3.A3': 232,\n",
       " 'B1.F#2.D3.B3': 233,\n",
       " 'B1.F#2.E-3': 234,\n",
       " 'B1.F#2.E-3.A3': 235,\n",
       " 'B1.F#2.E-3.B3': 236,\n",
       " 'B1.F#2.E3': 237,\n",
       " 'B1.F#2.E3.B3': 238,\n",
       " 'B1.G#2.D3': 239,\n",
       " 'B1.G#2.E-3': 240,\n",
       " 'B1.G#2.E3.G#3': 241,\n",
       " 'B1.G#2.F#3.G#3': 242,\n",
       " 'B1.G#2.F3': 243,\n",
       " 'B1.G#2.F3.C#4': 244,\n",
       " 'B1.G2': 245,\n",
       " 'B1.G2.C#3.B-3': 246,\n",
       " 'B2': 247,\n",
       " 'B2.A3': 248,\n",
       " 'B2.A3.E-4': 249,\n",
       " 'B2.A3.E4': 250,\n",
       " 'B2.B-3': 251,\n",
       " 'B2.B3': 252,\n",
       " 'B2.C#3': 253,\n",
       " 'B2.C#3.E-3': 254,\n",
       " 'B2.C3': 255,\n",
       " 'B2.C3.D3': 256,\n",
       " 'B2.C4': 257,\n",
       " 'B2.D3': 258,\n",
       " 'B2.D3.A3.D4': 259,\n",
       " 'B2.D3.B3.F#4': 260,\n",
       " 'B2.D3.G3': 261,\n",
       " 'B2.E-3': 262,\n",
       " 'B2.E-3.A3': 263,\n",
       " 'B2.E-3.B-3': 264,\n",
       " 'B2.E-3.E-4': 265,\n",
       " 'B2.E-3.E3.A3': 266,\n",
       " 'B2.E-3.F4': 267,\n",
       " 'B2.E-3.G#3': 268,\n",
       " 'B2.E3': 269,\n",
       " 'B2.E3.D4': 270,\n",
       " 'B2.E3.E4': 271,\n",
       " 'B2.F#3': 272,\n",
       " 'B2.F#3.A3': 273,\n",
       " 'B2.F#3.A3.B3.C4': 274,\n",
       " 'B2.F#3.B3': 275,\n",
       " 'B2.F#3.C#4': 276,\n",
       " 'B2.F#3.C4': 277,\n",
       " 'B2.F#3.D4': 278,\n",
       " 'B2.F#3.E-4': 279,\n",
       " 'B2.F#3.E-4.A4': 280,\n",
       " 'B2.F#3.E-4.B-4': 281,\n",
       " 'B2.F#3.E-4.B4': 282,\n",
       " 'B2.F#3.E-4.F#4': 283,\n",
       " 'B2.F#3.E4': 284,\n",
       " 'B2.F#3.G#3': 285,\n",
       " 'B2.F3': 286,\n",
       " 'B2.F3.E-4': 287,\n",
       " 'B2.F3.G#3': 288,\n",
       " 'B2.F3.G3': 289,\n",
       " 'B2.G#3': 290,\n",
       " 'B2.G#3.D4': 291,\n",
       " 'B2.G#3.E-4': 292,\n",
       " 'B2.G#3.E4': 293,\n",
       " 'B2.G#3.E4.B4': 294,\n",
       " 'B2.G#3.E4.F#4': 295,\n",
       " 'B2.G#3.E4.G#4': 296,\n",
       " 'B2.G#4': 297,\n",
       " 'B2.G3': 298,\n",
       " 'B2.G3.D4': 299,\n",
       " 'B2.G3.D4.F4': 300,\n",
       " 'B2.G3.E4': 301,\n",
       " 'B2.G3.E4.B4': 302,\n",
       " 'B3': 303,\n",
       " 'B3.A4': 304,\n",
       " 'B3.C#4': 305,\n",
       " 'B3.C#4.D4': 306,\n",
       " 'B3.C#4.D4.E4': 307,\n",
       " 'B3.C#4.E-4': 308,\n",
       " 'B3.C#4.E-4.E4': 309,\n",
       " 'B3.C#4.E-4.F4': 310,\n",
       " 'B3.C4': 311,\n",
       " 'B3.C4.D4': 312,\n",
       " 'B3.C4.E4.A4': 313,\n",
       " 'B3.C4.F4': 314,\n",
       " 'B3.D4': 315,\n",
       " 'B3.D4.F#4.G4': 316,\n",
       " 'B3.E-4': 317,\n",
       " 'B3.E-4.F#4': 318,\n",
       " 'B3.E4': 319,\n",
       " 'B3.F#4': 320,\n",
       " 'B3.F#4.A4': 321,\n",
       " 'B3.F#4.G#4': 322,\n",
       " 'B3.F#4.G4': 323,\n",
       " 'B3.F#4.G4.A4': 324,\n",
       " 'B3.F4': 325,\n",
       " 'B3.G#4': 326,\n",
       " 'B3.G#4.A4': 327,\n",
       " 'B3.G4': 328,\n",
       " 'B4': 329,\n",
       " 'B4.C#5': 330,\n",
       " 'B4.C5': 331,\n",
       " 'B4.C5.D5.E-5': 332,\n",
       " 'B4.D5': 333,\n",
       " 'C#2': 334,\n",
       " 'C#2.A2': 335,\n",
       " 'C#2.A2.E-3.C4': 336,\n",
       " 'C#2.A2.E3.G3': 337,\n",
       " 'C#2.A2.F#3.B3': 338,\n",
       " 'C#2.A2.F#3.C#4': 339,\n",
       " 'C#2.A2.G3': 340,\n",
       " 'C#2.B-2': 341,\n",
       " 'C#2.B-2.E3': 342,\n",
       " 'C#2.B-2.F#3': 343,\n",
       " 'C#2.B-2.F#3.B-3': 344,\n",
       " 'C#2.B-2.F3': 345,\n",
       " 'C#2.B-2.G#3.B-3': 346,\n",
       " 'C#2.B-2.G3': 347,\n",
       " 'C#2.B-2.G3.E-4': 348,\n",
       " 'C#2.B2': 349,\n",
       " 'C#2.B2.F3': 350,\n",
       " 'C#2.B2.F3.D4': 351,\n",
       " 'C#2.C#3': 352,\n",
       " 'C#2.C3.F#3.A3': 353,\n",
       " 'C#2.D3': 354,\n",
       " 'C#2.E3': 355,\n",
       " 'C#2.E3.G#3': 356,\n",
       " 'C#2.F#2.E3.B-3': 357,\n",
       " 'C#2.F2': 358,\n",
       " 'C#2.F2.C3': 359,\n",
       " 'C#2.F2.F3.G3': 360,\n",
       " 'C#2.F3': 361,\n",
       " 'C#2.F3.G#3': 362,\n",
       " 'C#2.G#2': 363,\n",
       " 'C#2.G#2.C#3': 364,\n",
       " 'C#2.G#2.C#3.F3': 365,\n",
       " 'C#2.G#2.E-3': 366,\n",
       " 'C#2.G#2.E3': 367,\n",
       " 'C#2.G#2.E3.B3': 368,\n",
       " 'C#2.G#2.E3.C#4': 369,\n",
       " 'C#2.G#2.E3.G#3': 370,\n",
       " 'C#2.G#2.F#3': 371,\n",
       " 'C#2.G#2.F#3.C#4': 372,\n",
       " 'C#2.G#2.F3': 373,\n",
       " 'C#2.G#2.F3.B3': 374,\n",
       " 'C#2.G#2.F3.C#4': 375,\n",
       " 'C#2.G#2.F3.C4': 376,\n",
       " 'C#3': 377,\n",
       " 'C#3.A3': 378,\n",
       " 'C#3.A3.E4': 379,\n",
       " 'C#3.B-3': 380,\n",
       " 'C#3.B-3.E4': 381,\n",
       " 'C#3.B-3.F#4': 382,\n",
       " 'C#3.B3': 383,\n",
       " 'C#3.B3.F4': 384,\n",
       " 'C#3.C#4': 385,\n",
       " 'C#3.C4': 386,\n",
       " 'C#3.C4.E-4': 387,\n",
       " 'C#3.C4.F4': 388,\n",
       " 'C#3.D3': 389,\n",
       " 'C#3.D3.E3': 390,\n",
       " 'C#3.D4': 391,\n",
       " 'C#3.E-3': 392,\n",
       " 'C#3.E-3.F3': 393,\n",
       " 'C#3.E-3.F3.F#3': 394,\n",
       " 'C#3.E-4': 395,\n",
       " 'C#3.E3': 396,\n",
       " 'C#3.E3.A3': 397,\n",
       " 'C#3.E3.B3.E4': 398,\n",
       " 'C#3.E3.C#4.G#4': 399,\n",
       " 'C#3.E3.G#3': 400,\n",
       " 'C#3.E4': 401,\n",
       " 'C#3.F#3': 402,\n",
       " 'C#3.F#3.E4': 403,\n",
       " 'C#3.F#3.F#4': 404,\n",
       " 'C#3.F#3.G#3': 405,\n",
       " 'C#3.F#3.G3': 406,\n",
       " 'C#3.F#4': 407,\n",
       " 'C#3.F3': 408,\n",
       " 'C#3.F3.B-3': 409,\n",
       " 'C#3.F3.B3': 410,\n",
       " 'C#3.F3.C4': 411,\n",
       " 'C#3.F3.F#3': 412,\n",
       " 'C#3.F3.F4': 413,\n",
       " 'C#3.F3.G4': 414,\n",
       " 'C#3.F4': 415,\n",
       " 'C#3.G#3': 416,\n",
       " 'C#3.G#3.B-3': 417,\n",
       " 'C#3.G#3.B-3.C4': 418,\n",
       " 'C#3.G#3.B3': 419,\n",
       " 'C#3.G#3.B3.C#4.D4': 420,\n",
       " 'C#3.G#3.C#4': 421,\n",
       " 'C#3.G#3.D4': 422,\n",
       " 'C#3.G#3.E4': 423,\n",
       " 'C#3.G#3.F#4': 424,\n",
       " 'C#3.G#3.F4': 425,\n",
       " 'C#3.G#3.F4.B4': 426,\n",
       " 'C#3.G#4': 427,\n",
       " 'C#3.G3': 428,\n",
       " 'C#3.G3.A3': 429,\n",
       " 'C#3.G3.A3.B-3': 430,\n",
       " 'C#3.G3.B-3': 431,\n",
       " 'C#4': 432,\n",
       " 'C#4.A4': 433,\n",
       " 'C#4.B-4': 434,\n",
       " 'C#4.B-4.B4': 435,\n",
       " 'C#4.B4': 436,\n",
       " 'C#4.D4': 437,\n",
       " 'C#4.D4.E4': 438,\n",
       " 'C#4.D4.F#4.B4': 439,\n",
       " 'C#4.E-4': 440,\n",
       " 'C#4.E-4.E4': 441,\n",
       " 'C#4.E-4.E4.F#4': 442,\n",
       " 'C#4.E-4.F4': 443,\n",
       " 'C#4.E-4.F4.F#4': 444,\n",
       " 'C#4.E-4.F4.G4': 445,\n",
       " 'C#4.E4': 446,\n",
       " 'C#4.E4.G#4.A4': 447,\n",
       " 'C#4.F#4': 448,\n",
       " 'C#4.F4': 449,\n",
       " 'C#4.F4.G4.G#4': 450,\n",
       " 'C#4.G#4': 451,\n",
       " 'C#4.G#4.A4': 452,\n",
       " 'C#4.G#4.A4.B4': 453,\n",
       " 'C#4.G4': 454,\n",
       " 'C#5': 455,\n",
       " 'C#5.D5': 456,\n",
       " 'C#5.E-5': 457,\n",
       " 'C#5.E5': 458,\n",
       " 'C2': 459,\n",
       " 'C2.A2': 460,\n",
       " 'C2.A2.E-3': 461,\n",
       " 'C2.A2.E3': 462,\n",
       " 'C2.A2.F#3': 463,\n",
       " 'C2.A2.F#3.D4': 464,\n",
       " 'C2.A2.F3': 465,\n",
       " 'C2.A2.F3.A3': 466,\n",
       " 'C2.A2.G3.A3': 467,\n",
       " 'C2.B-2': 468,\n",
       " 'C2.B-2.E3': 469,\n",
       " 'C2.B-2.E3.C#4': 470,\n",
       " 'C2.B2.F3.G#3': 471,\n",
       " 'C2.C#3': 472,\n",
       " 'C2.C3': 473,\n",
       " 'C2.E-3': 474,\n",
       " 'C2.E-3.G3': 475,\n",
       " 'C2.E3.G3': 476,\n",
       " 'C2.F2.E-3.A3': 477,\n",
       " 'C2.G#2': 478,\n",
       " 'C2.G#2.D3.B3': 479,\n",
       " 'C2.G#2.E-3.F#3': 480,\n",
       " 'C2.G#2.F#3': 481,\n",
       " 'C2.G#2.F3.B-3': 482,\n",
       " 'C2.G#2.F3.C4': 483,\n",
       " 'C2.G2': 484,\n",
       " 'C2.G2.C3': 485,\n",
       " 'C2.G2.D3': 486,\n",
       " 'C2.G2.E-3': 487,\n",
       " 'C2.G2.E-3.B-3': 488,\n",
       " 'C2.G2.E-3.C4': 489,\n",
       " 'C2.G2.E-3.G3': 490,\n",
       " 'C2.G2.E3': 491,\n",
       " 'C2.G2.E3.B-3': 492,\n",
       " 'C2.G2.E3.B3': 493,\n",
       " 'C2.G2.E3.C4': 494,\n",
       " 'C2.G2.F3': 495,\n",
       " 'C2.G2.F3.C4': 496,\n",
       " 'C3': 497,\n",
       " 'C3.A3': 498,\n",
       " 'C3.A3.E-4': 499,\n",
       " 'C3.A3.E4': 500,\n",
       " 'C3.A3.F4': 501,\n",
       " 'C3.B-3': 502,\n",
       " 'C3.B-3.E4': 503,\n",
       " 'C3.B3': 504,\n",
       " 'C3.B3.D4': 505,\n",
       " 'C3.B3.E4': 506,\n",
       " 'C3.C#3': 507,\n",
       " 'C3.C#3.E-3': 508,\n",
       " 'C3.C#4': 509,\n",
       " 'C3.C4': 510,\n",
       " 'C3.D3': 511,\n",
       " 'C3.D3.E3': 512,\n",
       " 'C3.D3.E3.F3': 513,\n",
       " 'C3.D4': 514,\n",
       " 'C3.E-3': 515,\n",
       " 'C3.E-3.B-3.E-4': 516,\n",
       " 'C3.E-3.C4.G4': 517,\n",
       " 'C3.E-3.G#3': 518,\n",
       " 'C3.E-3.G3': 519,\n",
       " 'C3.E-4': 520,\n",
       " 'C3.E3': 521,\n",
       " 'C3.E3.A3': 522,\n",
       " 'C3.E3.B-3': 523,\n",
       " 'C3.E3.B3': 524,\n",
       " 'C3.E3.E4': 525,\n",
       " 'C3.E3.F#4': 526,\n",
       " 'C3.E3.F3.B-3': 527,\n",
       " 'C3.E4': 528,\n",
       " 'C3.F#3': 529,\n",
       " 'C3.F#3.A3': 530,\n",
       " 'C3.F#3.E4': 531,\n",
       " 'C3.F#3.G#3': 532,\n",
       " 'C3.F3': 533,\n",
       " 'C3.F3.E-4': 534,\n",
       " 'C3.F3.F#3': 535,\n",
       " 'C3.F3.F4': 536,\n",
       " 'C3.F3.G3': 537,\n",
       " 'C3.F4': 538,\n",
       " 'C3.G#3': 539,\n",
       " 'C3.G#3.E-4': 540,\n",
       " 'C3.G#3.E-4.F#4': 541,\n",
       " 'C3.G3': 542,\n",
       " 'C3.G3.A3': 543,\n",
       " 'C3.G3.A3.B3': 544,\n",
       " 'C3.G3.B-3': 545,\n",
       " 'C3.G3.B-3.C4.C#4': 546,\n",
       " 'C3.G3.C#4': 547,\n",
       " 'C3.G3.C4': 548,\n",
       " 'C3.G3.D4': 549,\n",
       " 'C3.G3.E-4': 550,\n",
       " 'C3.G3.E4': 551,\n",
       " 'C3.G3.E4.B-4': 552,\n",
       " 'C3.G3.F4': 553,\n",
       " 'C3.G4': 554,\n",
       " 'C4': 555,\n",
       " 'C4.A4': 556,\n",
       " 'C4.A4.B-4': 557,\n",
       " 'C4.B-4': 558,\n",
       " 'C4.C#4': 559,\n",
       " 'C4.C#4.E-4': 560,\n",
       " 'C4.C#4.F#4': 561,\n",
       " 'C4.C#4.F4.B-4': 562,\n",
       " 'C4.D4': 563,\n",
       " 'C4.D4.E-4': 564,\n",
       " 'C4.D4.E-4.F4': 565,\n",
       " 'C4.D4.E4': 566,\n",
       " 'C4.D4.E4.F#4': 567,\n",
       " 'C4.D4.E4.F4': 568,\n",
       " 'C4.E-4': 569,\n",
       " 'C4.E-4.G4.G#4': 570,\n",
       " 'C4.E4': 571,\n",
       " 'C4.E4.F#4.G4': 572,\n",
       " 'C4.F#4': 573,\n",
       " 'C4.F4': 574,\n",
       " 'C4.G#4': 575,\n",
       " 'C4.G4': 576,\n",
       " 'C4.G4.G#4': 577,\n",
       " 'C4.G4.G#4.B-4': 578,\n",
       " 'C5': 579,\n",
       " 'C5.C#5': 580,\n",
       " 'C5.C#5.E-5.E5': 581,\n",
       " 'C5.D5': 582,\n",
       " 'C5.E-5': 583,\n",
       " 'D2': 584,\n",
       " 'D2.A2': 585,\n",
       " 'D2.A2.D3': 586,\n",
       " 'D2.A2.D3.F#3': 587,\n",
       " 'D2.A2.E3': 588,\n",
       " 'D2.A2.F#3': 589,\n",
       " 'D2.A2.F#3.C#4': 590,\n",
       " 'D2.A2.F#3.C4': 591,\n",
       " 'D2.A2.F#3.D4': 592,\n",
       " 'D2.A2.F3': 593,\n",
       " 'D2.A2.F3.A3': 594,\n",
       " 'D2.A2.F3.C4': 595,\n",
       " 'D2.A2.F3.D4': 596,\n",
       " 'D2.A2.G3': 597,\n",
       " 'D2.A2.G3.D4': 598,\n",
       " 'D2.B-2': 599,\n",
       " 'D2.B-2.E3.C#4': 600,\n",
       " 'D2.B-2.F3': 601,\n",
       " 'D2.B-2.F3.G#3': 602,\n",
       " 'D2.B-2.G#3': 603,\n",
       " 'D2.B-2.G3.C4': 604,\n",
       " 'D2.B-2.G3.D4': 605,\n",
       " 'D2.B2': 606,\n",
       " 'D2.B2.A3.B3': 607,\n",
       " 'D2.B2.F#3': 608,\n",
       " 'D2.B2.F3': 609,\n",
       " 'D2.B2.G#3': 610,\n",
       " 'D2.B2.G#3.E4': 611,\n",
       " 'D2.B2.G3': 612,\n",
       " 'D2.B2.G3.B3': 613,\n",
       " 'D2.C#3.G3.B-3': 614,\n",
       " 'D2.C3': 615,\n",
       " 'D2.C3.F#3': 616,\n",
       " 'D2.C3.F#3.E-4': 617,\n",
       " 'D2.D3': 618,\n",
       " 'D2.D3.F3': 619,\n",
       " 'D2.E-3': 620,\n",
       " 'D2.F#2': 621,\n",
       " 'D2.F#2.C#3': 622,\n",
       " 'D2.F#2.F#3.G#3': 623,\n",
       " 'D2.F#3': 624,\n",
       " 'D2.F#3.A3': 625,\n",
       " 'D2.F2': 626,\n",
       " 'D2.F3': 627,\n",
       " 'D2.F3.A3': 628,\n",
       " 'D2.G2.F3.B3': 629,\n",
       " 'D3': 630,\n",
       " 'D3.A3': 631,\n",
       " 'D3.A3.B3': 632,\n",
       " 'D3.A3.B3.C#4': 633,\n",
       " 'D3.A3.C4': 634,\n",
       " 'D3.A3.D4': 635,\n",
       " 'D3.A3.E4': 636,\n",
       " 'D3.A3.F#4': 637,\n",
       " 'D3.A3.F4': 638,\n",
       " 'D3.A3.G4': 639,\n",
       " 'D3.A4': 640,\n",
       " 'D3.B-3': 641,\n",
       " 'D3.B-3.F4': 642,\n",
       " 'D3.B3': 643,\n",
       " 'D3.B3.C4': 644,\n",
       " 'D3.B3.C4.D4': 645,\n",
       " 'D3.B3.F4': 646,\n",
       " 'D3.B3.G4': 647,\n",
       " 'D3.C#4': 648,\n",
       " 'D3.C#4.E4': 649,\n",
       " 'D3.C#4.F#4': 650,\n",
       " 'D3.C4': 651,\n",
       " 'D3.C4.D4': 652,\n",
       " 'D3.C4.F#4': 653,\n",
       " 'D3.C4.G4.A4': 654,\n",
       " 'D3.D4': 655,\n",
       " 'D3.E-3': 656,\n",
       " 'D3.E-3.F3': 657,\n",
       " 'D3.E-4': 658,\n",
       " 'D3.E3': 659,\n",
       " 'D3.E3.F#3': 660,\n",
       " 'D3.E3.F#3.G3': 661,\n",
       " 'D3.E3.F3.A3': 662,\n",
       " 'D3.E4': 663,\n",
       " 'D3.F#3': 664,\n",
       " 'D3.F#3.B3': 665,\n",
       " 'D3.F#3.C#4': 666,\n",
       " 'D3.F#3.C4': 667,\n",
       " 'D3.F#3.F#4': 668,\n",
       " 'D3.F#3.G#4': 669,\n",
       " 'D3.F#3.G3': 670,\n",
       " 'D3.F#4': 671,\n",
       " 'D3.F3': 672,\n",
       " 'D3.F3.A3': 673,\n",
       " 'D3.F3.B-3': 674,\n",
       " 'D3.F3.C4.F4': 675,\n",
       " 'D3.F4': 676,\n",
       " 'D3.G#3': 677,\n",
       " 'D3.G#3.B-3': 678,\n",
       " 'D3.G#3.B-3.B3': 679,\n",
       " 'D3.G#3.B3': 680,\n",
       " 'D3.G3': 681,\n",
       " 'D3.G3.A3': 682,\n",
       " 'D3.G3.G#3': 683,\n",
       " 'D3.G4': 684,\n",
       " 'D4': 685,\n",
       " 'D4.A4': 686,\n",
       " 'D4.B-4': 687,\n",
       " 'D4.B4': 688,\n",
       " 'D4.C5': 689,\n",
       " 'D4.E-4': 690,\n",
       " 'D4.E-4.F4': 691,\n",
       " 'D4.E-4.G4.C5': 692,\n",
       " 'D4.E4': 693,\n",
       " 'D4.E4.F#4': 694,\n",
       " 'D4.E4.F#4.G#4': 695,\n",
       " 'D4.E4.F#4.G4': 696,\n",
       " 'D4.E4.F4': 697,\n",
       " 'D4.E4.F4.G4': 698,\n",
       " 'D4.F#4': 699,\n",
       " 'D4.F#4.G#4.A4': 700,\n",
       " 'D4.F4': 701,\n",
       " 'D4.F4.A4.B-4': 702,\n",
       " 'D4.G#4': 703,\n",
       " 'D4.G4': 704,\n",
       " 'D5': 705,\n",
       " 'E-2': 706,\n",
       " 'E-2.A2.F3': 707,\n",
       " 'E-2.B-2': 708,\n",
       " 'E-2.B-2.C#3': 709,\n",
       " 'E-2.B-2.C3': 710,\n",
       " 'E-2.B-2.E-3': 711,\n",
       " 'E-2.B-2.E-3.G3': 712,\n",
       " 'E-2.B-2.F#3': 713,\n",
       " 'E-2.B-2.F#3.B-3': 714,\n",
       " 'E-2.B-2.F#3.E-4': 715,\n",
       " 'E-2.B-2.F3': 716,\n",
       " 'E-2.B-2.G#3': 717,\n",
       " 'E-2.B-2.G3': 718,\n",
       " 'E-2.B-2.G3.C#4': 719,\n",
       " 'E-2.B-2.G3.D4': 720,\n",
       " 'E-2.B-2.G3.E-4': 721,\n",
       " 'E-2.B2.A3': 722,\n",
       " 'E-2.B2.F#3': 723,\n",
       " 'E-2.B2.F#3.A3': 724,\n",
       " 'E-2.B2.G#3.C#4': 725,\n",
       " 'E-2.B2.G#3.E-4': 726,\n",
       " 'E-2.C#3.G3.E4': 727,\n",
       " 'E-2.C3': 728,\n",
       " 'E-2.C3.A3': 729,\n",
       " 'E-2.C3.F#3': 730,\n",
       " 'E-2.C3.G#3': 731,\n",
       " 'E-2.C3.G3': 732,\n",
       " 'E-2.E-3.F#3': 733,\n",
       " 'E-2.E3': 734,\n",
       " 'E-2.F#2': 735,\n",
       " 'E-2.F#2.C#3': 736,\n",
       " 'E-2.F#2.C3': 737,\n",
       " 'E-2.F2': 738,\n",
       " 'E-2.F2.C3.A3': 739,\n",
       " 'E-2.G#2': 740,\n",
       " 'E-2.G#2.F#3.C4': 741,\n",
       " 'E-2.G2': 742,\n",
       " 'E-2.G2.C3': 743,\n",
       " 'E-2.G2.C3.G3': 744,\n",
       " 'E-2.G2.D3': 745,\n",
       " 'E-2.G2.G3.A3': 746,\n",
       " 'E-2.G3': 747,\n",
       " 'E-3': 748,\n",
       " 'E-3.A3': 749,\n",
       " 'E-3.A3.B3': 750,\n",
       " 'E-3.A3.B3.C4': 751,\n",
       " 'E-3.A3.C4': 752,\n",
       " 'E-3.B-3': 753,\n",
       " 'E-3.B-3.C4.D4': 754,\n",
       " 'E-3.B-3.F#4': 755,\n",
       " 'E-3.B-3.F4': 756,\n",
       " 'E-3.B-3.G#4': 757,\n",
       " 'E-3.B-3.G4': 758,\n",
       " 'E-3.B-4': 759,\n",
       " 'E-3.B3': 760,\n",
       " 'E-3.C#4': 761,\n",
       " 'E-3.C#4.E-4': 762,\n",
       " 'E-3.C#4.G#4.B-4': 763,\n",
       " 'E-3.C#4.G4': 764,\n",
       " 'E-3.C4': 765,\n",
       " 'E-3.C4.C#4': 766,\n",
       " 'E-3.C4.C#4.E-4': 767,\n",
       " 'E-3.C4.F#4': 768,\n",
       " 'E-3.C4.F#4.A4': 769,\n",
       " 'E-3.C4.G#4': 770,\n",
       " 'E-3.D4': 771,\n",
       " 'E-3.D4.F4': 772,\n",
       " 'E-3.D4.G4': 773,\n",
       " 'E-3.E-4': 774,\n",
       " 'E-3.E3': 775,\n",
       " 'E-3.E3.F#3': 776,\n",
       " 'E-3.E4': 777,\n",
       " 'E-3.F#3': 778,\n",
       " 'E-3.F#3.B-3': 779,\n",
       " 'E-3.F#3.B3': 780,\n",
       " 'E-3.F#3.C#4.F#4': 781,\n",
       " 'E-3.F#4': 782,\n",
       " 'E-3.F3': 783,\n",
       " 'E-3.F3.F#3.B-3': 784,\n",
       " 'E-3.F3.G3': 785,\n",
       " 'E-3.F3.G3.G#3': 786,\n",
       " 'E-3.F4': 787,\n",
       " 'E-3.G#3': 788,\n",
       " 'E-3.G#3.A3': 789,\n",
       " 'E-3.G#3.B-3': 790,\n",
       " 'E-3.G#4': 791,\n",
       " 'E-3.G3': 792,\n",
       " 'E-3.G3.G#3': 793,\n",
       " 'E-3.G4': 794,\n",
       " 'E-4': 795,\n",
       " 'E-4.A4': 796,\n",
       " 'E-4.B-4': 797,\n",
       " 'E-4.B4': 798,\n",
       " 'E-4.C#5': 799,\n",
       " 'E-4.C5': 800,\n",
       " 'E-4.E4': 801,\n",
       " 'E-4.E4.F#4': 802,\n",
       " 'E-4.E4.G#4.C#5': 803,\n",
       " 'E-4.F#4': 804,\n",
       " 'E-4.F#4.B-4.B4': 805,\n",
       " 'E-4.F4': 806,\n",
       " 'E-4.F4.F#4': 807,\n",
       " 'E-4.F4.F#4.G#4': 808,\n",
       " 'E-4.F4.G4': 809,\n",
       " 'E-4.F4.G4.G#4': 810,\n",
       " 'E-4.G#4': 811,\n",
       " 'E-4.G4': 812,\n",
       " 'E-4.G4.A4.B-4': 813,\n",
       " 'E-5': 814,\n",
       " 'E2': 815,\n",
       " 'E2.A2': 816,\n",
       " 'E2.A2.G3.C#4': 817,\n",
       " 'E2.B-2.C#3.B-3': 818,\n",
       " 'E2.B-2.F#3': 819,\n",
       " 'E2.B2': 820,\n",
       " 'E2.B2.A3': 821,\n",
       " 'E2.B2.C#3': 822,\n",
       " 'E2.B2.D3': 823,\n",
       " 'E2.B2.E3': 824,\n",
       " 'E2.B2.E3.G#3': 825,\n",
       " 'E2.B2.F#3': 826,\n",
       " 'E2.B2.G#3': 827,\n",
       " 'E2.B2.G#3.D4': 828,\n",
       " 'E2.B2.G#3.E-4': 829,\n",
       " 'E2.B2.G#3.E4': 830,\n",
       " 'E2.B2.G3': 831,\n",
       " 'E2.B2.G3.B3': 832,\n",
       " 'E2.B2.G3.E4': 833,\n",
       " 'E2.C#3': 834,\n",
       " 'E2.C#3.A3': 835,\n",
       " 'E2.C#3.G3': 836,\n",
       " 'E2.C3.A3.D4': 837,\n",
       " 'E2.C3.A3.E4': 838,\n",
       " 'E2.C3.B-3': 839,\n",
       " 'E2.C3.G3': 840,\n",
       " 'E2.C3.G3.B-3': 841,\n",
       " 'E2.D3.B3.D4': 842,\n",
       " 'E2.D3.G#3.F4': 843,\n",
       " 'E2.E3.G3': 844,\n",
       " 'E2.F#2': 845,\n",
       " 'E2.F#2.C#3.B-3': 846,\n",
       " 'E2.F2': 847,\n",
       " 'E2.F3': 848,\n",
       " 'E2.G#2': 849,\n",
       " 'E2.G#2.C#3': 850,\n",
       " 'E2.G#2.C#3.G#3': 851,\n",
       " 'E2.G#2.E-3': 852,\n",
       " 'E2.G#2.G#3.B-3': 853,\n",
       " 'E2.G#3': 854,\n",
       " 'E2.G2': 855,\n",
       " 'E2.G2.C#3': 856,\n",
       " 'E2.G2.C3.G3': 857,\n",
       " 'E2.G2.D3': 858,\n",
       " 'E2.G3': 859,\n",
       " 'E3': 860,\n",
       " 'E3.A3': 861,\n",
       " 'E3.A3.B-3': 862,\n",
       " 'E3.A3.B3': 863,\n",
       " 'E3.A3.G4': 864,\n",
       " 'E3.A4': 865,\n",
       " 'E3.B-3': 866,\n",
       " 'E3.B-3.C4.C#4': 867,\n",
       " 'E3.B3': 868,\n",
       " 'E3.B3.A4': 869,\n",
       " 'E3.B3.C#4.E-4': 870,\n",
       " 'E3.B3.D4': 871,\n",
       " 'E3.B3.F#4': 872,\n",
       " 'E3.B3.G#4': 873,\n",
       " 'E3.B3.G4': 874,\n",
       " 'E3.B4': 875,\n",
       " 'E3.C#4': 876,\n",
       " 'E3.C#4.A4': 877,\n",
       " 'E3.C#4.D4': 878,\n",
       " 'E3.C#4.D4.E4': 879,\n",
       " 'E3.C#4.G4': 880,\n",
       " 'E3.C#4.G4.B-4': 881,\n",
       " 'E3.C4': 882,\n",
       " 'E3.C4.G4': 883,\n",
       " 'E3.D4': 884,\n",
       " 'E3.D4.A4.B4': 885,\n",
       " 'E3.D4.E4': 886,\n",
       " 'E3.D4.G#4': 887,\n",
       " 'E3.E-4': 888,\n",
       " 'E3.E-4.F#4': 889,\n",
       " 'E3.E-4.G#4': 890,\n",
       " 'E3.E4': 891,\n",
       " 'E3.F#3': 892,\n",
       " 'E3.F#3.G#3': 893,\n",
       " 'E3.F#3.G#3.A3': 894,\n",
       " 'E3.F#3.G3.B3': 895,\n",
       " 'E3.F#4': 896,\n",
       " 'E3.F3': 897,\n",
       " 'E3.F3.G3': 898,\n",
       " 'E3.F4': 899,\n",
       " 'E3.G#3': 900,\n",
       " 'E3.G#3.A3': 901,\n",
       " 'E3.G#4': 902,\n",
       " 'E3.G3': 903,\n",
       " 'E3.G3.B3': 904,\n",
       " 'E3.G3.D4': 905,\n",
       " 'E3.G4': 906,\n",
       " 'E4': 907,\n",
       " 'E4.A4': 908,\n",
       " 'E4.B-4': 909,\n",
       " 'E4.B4': 910,\n",
       " 'E4.C#5': 911,\n",
       " 'E4.C5': 912,\n",
       " 'E4.D5': 913,\n",
       " 'E4.F#4': 914,\n",
       " 'E4.F#4.G#4': 915,\n",
       " 'E4.F#4.G#4.A4': 916,\n",
       " 'E4.F#4.G4': 917,\n",
       " 'E4.F#4.G4.A4': 918,\n",
       " 'E4.F4': 919,\n",
       " 'E4.F4.A4': 920,\n",
       " 'E4.F4.G4': 921,\n",
       " 'E4.G#4': 922,\n",
       " 'E4.G#4.B-4.B4': 923,\n",
       " 'E4.G4': 924,\n",
       " 'E5': 925,\n",
       " 'F#2': 926,\n",
       " 'F#2.A2': 927,\n",
       " 'F#2.A2.D3.A3': 928,\n",
       " 'F#2.A2.E-3': 929,\n",
       " 'F#2.A2.E3': 930,\n",
       " 'F#2.A3': 931,\n",
       " 'F#2.B-2': 932,\n",
       " 'F#2.B-2.E-3': 933,\n",
       " 'F#2.B-2.E-3.B-3': 934,\n",
       " 'F#2.B-2.F3': 935,\n",
       " 'F#2.B-3': 936,\n",
       " 'F#2.B-3.C#4': 937,\n",
       " 'F#2.B-3.F#4': 938,\n",
       " 'F#2.B2': 939,\n",
       " 'F#2.B3': 940,\n",
       " 'F#2.C#3': 941,\n",
       " 'F#2.C#3.A3': 942,\n",
       " 'F#2.C#3.B-3': 943,\n",
       " 'F#2.C#3.B-3.E4': 944,\n",
       " 'F#2.C#3.B-3.F4': 945,\n",
       " 'F#2.C#3.B-3.G#4': 946,\n",
       " 'F#2.C#3.B3': 947,\n",
       " 'F#2.C#3.B3.F#4': 948,\n",
       " 'F#2.C#3.C#4': 949,\n",
       " 'F#2.C#3.C4': 950,\n",
       " 'F#2.C#3.E-3': 951,\n",
       " 'F#2.C#3.E3': 952,\n",
       " 'F#2.C#3.F#3': 953,\n",
       " 'F#2.C#3.G#3': 954,\n",
       " 'F#2.C#3.G#3.F#4': 955,\n",
       " 'F#2.C3': 956,\n",
       " 'F#2.C3.A3': 957,\n",
       " 'F#2.C3.E-3.C4': 958,\n",
       " 'F#2.C3.G#3': 959,\n",
       " 'F#2.D3': 960,\n",
       " 'F#2.D3.A3': 961,\n",
       " 'F#2.D3.A3.C4': 962,\n",
       " 'F#2.D3.B3': 963,\n",
       " 'F#2.D3.C4': 964,\n",
       " 'F#2.D3.C4.A4': 965,\n",
       " 'F#2.E-3': 966,\n",
       " 'F#2.E-3.A3': 967,\n",
       " 'F#2.E-3.B-3': 968,\n",
       " 'F#2.E-3.B3': 969,\n",
       " 'F#2.E-3.C#4': 970,\n",
       " 'F#2.E-3.C4': 971,\n",
       " 'F#2.E-3.C4.G#4': 972,\n",
       " 'F#2.E-3.E3': 973,\n",
       " 'F#2.E3': 974,\n",
       " 'F#2.E3.A3': 975,\n",
       " 'F#2.E3.B-3': 976,\n",
       " 'F#2.E3.B3': 977,\n",
       " 'F#2.E3.C#4': 978,\n",
       " 'F#2.E3.C#4.E4': 979,\n",
       " 'F#2.E3.F#3': 980,\n",
       " 'F#2.E4': 981,\n",
       " 'F#2.F#3': 982,\n",
       " 'F#2.F#3.A3': 983,\n",
       " 'F#2.F#3.B-3.C4': 984,\n",
       " 'F#2.F#3.B-3.E-4': 985,\n",
       " 'F#2.F#3.E-4': 986,\n",
       " 'F#2.F3': 987,\n",
       " 'F#2.F3.C#4': 988,\n",
       " 'F#2.F3.D4': 989,\n",
       " 'F#2.G#2': 990,\n",
       " 'F#2.G#2.E-3.C4': 991,\n",
       " 'F#2.G#3': 992,\n",
       " 'F#2.G2': 993,\n",
       " 'F#2.G2.A2.B2': 994,\n",
       " 'F#2.G3': 995,\n",
       " 'F#3': 996,\n",
       " 'F#3.A3': 997,\n",
       " 'F#3.A3.E4': 998,\n",
       " 'F#3.A4': 999,\n",
       " ...}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('\\nnote_to_int')\n",
    "note_to_int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "duration_to_int\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{0: 0,\n",
       " Fraction(1, 12): 1,\n",
       " Fraction(1, 6): 2,\n",
       " 0.25: 3,\n",
       " Fraction(1, 3): 4,\n",
       " 0.5: 5,\n",
       " Fraction(2, 3): 6,\n",
       " 0.75: 7,\n",
       " 1.0: 8,\n",
       " 1.25: 9,\n",
       " Fraction(4, 3): 10,\n",
       " 1.5: 11,\n",
       " 1.75: 12,\n",
       " 2.0: 13,\n",
       " 2.5: 14,\n",
       " 3.0: 15,\n",
       " 4.0: 16}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('\\nduration_to_int')\n",
    "duration_to_int"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the sequences used by the Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "network_input, network_output = prepare_sequences(notes, durations, lookups, distincts, seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pitch input\n",
      "[1388 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388\n",
      " 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388 1388\n",
      " 1388 1388 1388 1388]\n",
      "duration input\n",
      "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      "pitch output\n",
      "[0. 0. 0. ... 0. 0. 0.]\n",
      "duration output\n",
      "[0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.]\n"
     ]
    }
   ],
   "source": [
    "print('pitch input')\n",
    "print(network_input[0][0])\n",
    "print('duration input')\n",
    "print(network_input[1][0])\n",
    "print('pitch output')\n",
    "print(network_output[0][0])\n",
    "print('duration output')\n",
    "print(network_output[1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1388, 1388, 1388, ..., 1388, 1388, 1388],\n",
       "       [1388, 1388, 1388, ..., 1388, 1388, 1112],\n",
       "       [1388, 1388, 1388, ..., 1388, 1112, 1346],\n",
       "       ...,\n",
       "       [ 497,    0,  497, ...,   74, 1346, 1112],\n",
       "       [   0,  497, 1045, ..., 1346, 1112,  497],\n",
       "       [ 497, 1045,  497, ..., 1112,  497,  907]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "network_input[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create the structure of the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-19 07:45:41.359878: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-01-19 07:45:41.945792: W tensorflow/core/common_runtime/gpu/gpu_bfc_allocator.cc:42] Overriding orig_value setting because the TF_FORCE_GPU_ALLOW_GROWTH environment variable is set. Original config value was 0.\n",
      "2024-01-19 07:45:41.945856: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1613] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 78977 MB memory:  -> device: 0, name: NVIDIA A100-SXM4-80GB, pci bus id: 0000:01:00.0, compute capability: 8.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      " Layer (type)                   Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      " input_1 (InputLayer)           [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " input_2 (InputLayer)           [(None, None)]       0           []                               \n",
      "                                                                                                  \n",
      " embedding (Embedding)          (None, None, 100)    138900      ['input_1[0][0]']                \n",
      "                                                                                                  \n",
      " embedding_1 (Embedding)        (None, None, 100)    1700        ['input_2[0][0]']                \n",
      "                                                                                                  \n",
      " concatenate (Concatenate)      (None, None, 200)    0           ['embedding[0][0]',              \n",
      "                                                                  'embedding_1[0][0]']            \n",
      "                                                                                                  \n",
      " lstm (LSTM)                    (None, None, 128)    168448      ['concatenate[0][0]']            \n",
      "                                                                                                  \n",
      " dropout (Dropout)              (None, None, 128)    0           ['lstm[0][0]']                   \n",
      "                                                                                                  \n",
      " lstm_1 (LSTM)                  (None, None, 128)    131584      ['dropout[0][0]']                \n",
      "                                                                                                  \n",
      " dropout_1 (Dropout)            (None, None, 128)    0           ['lstm_1[0][0]']                 \n",
      "                                                                                                  \n",
      " dense (Dense)                  (None, None, 1)      129         ['dropout_1[0][0]']              \n",
      "                                                                                                  \n",
      " reshape (Reshape)              (None, None)         0           ['dense[0][0]']                  \n",
      "                                                                                                  \n",
      " activation (Activation)        (None, None)         0           ['reshape[0][0]']                \n",
      "                                                                                                  \n",
      " repeat_vector (RepeatVector)   (None, 128, None)    0           ['activation[0][0]']             \n",
      "                                                                                                  \n",
      " permute (Permute)              (None, None, 128)    0           ['repeat_vector[0][0]']          \n",
      "                                                                                                  \n",
      " multiply (Multiply)            (None, None, 128)    0           ['dropout_1[0][0]',              \n",
      "                                                                  'permute[0][0]']                \n",
      "                                                                                                  \n",
      " lambda (Lambda)                (None, 128)          0           ['multiply[0][0]']               \n",
      "                                                                                                  \n",
      " pitch (Dense)                  (None, 1389)         179181      ['lambda[0][0]']                 \n",
      "                                                                                                  \n",
      " duration (Dense)               (None, 17)           2193        ['lambda[0][0]']                 \n",
      "                                                                                                  \n",
      "==================================================================================================\n",
      "Total params: 622,135\n",
      "Trainable params: 622,135\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/keras/optimizers/optimizer_v2/rmsprop.py:143: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  super().__init__(name, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "model, att_model = create_network(n_notes, n_durations, embed_size, rnn_units, use_attention)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install pydot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "plot_model(model, to_file=os.path.join(run_folder ,'viz/model.png'), show_shapes = True, show_layer_names = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the neural network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "weights_folder = os.path.join(run_folder, 'weights')\n",
    "# model.load_weights(os.path.join(weights_folder, \"weights.h5\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/2000000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-01-19 07:47:32.507628: I tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:428] Loaded cuDNN version 8500\n",
      "2024-01-19 07:47:32.702844: I tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:630] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1786/1786 [==============================] - 28s 14ms/step - loss: 4.2169 - pitch_loss: 3.6157 - duration_loss: 0.6012 - val_loss: 4.1094 - val_pitch_loss: 3.3633 - val_duration_loss: 0.7461\n",
      "Epoch 2/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 3.6613 - pitch_loss: 3.1984 - duration_loss: 0.4630 - val_loss: 3.9246 - val_pitch_loss: 3.2193 - val_duration_loss: 0.7052\n",
      "Epoch 3/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 3.5159 - pitch_loss: 3.0923 - duration_loss: 0.4237 - val_loss: 3.9032 - val_pitch_loss: 3.2130 - val_duration_loss: 0.6902\n",
      "Epoch 4/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 3.4152 - pitch_loss: 3.0223 - duration_loss: 0.3929 - val_loss: 3.9143 - val_pitch_loss: 3.2000 - val_duration_loss: 0.7143\n",
      "Epoch 5/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 3.3219 - pitch_loss: 2.9621 - duration_loss: 0.3598 - val_loss: 3.9387 - val_pitch_loss: 3.2043 - val_duration_loss: 0.7344\n",
      "Epoch 6/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 3.2514 - pitch_loss: 2.9226 - duration_loss: 0.3288 - val_loss: 4.0040 - val_pitch_loss: 3.2308 - val_duration_loss: 0.7732\n",
      "Epoch 7/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 3.2091 - pitch_loss: 2.9096 - duration_loss: 0.2994 - val_loss: 4.1191 - val_pitch_loss: 3.2513 - val_duration_loss: 0.8677\n",
      "Epoch 8/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 3.1922 - pitch_loss: 2.9190 - duration_loss: 0.2732 - val_loss: 4.2331 - val_pitch_loss: 3.3226 - val_duration_loss: 0.9105\n",
      "Epoch 9/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 3.1862 - pitch_loss: 2.9363 - duration_loss: 0.2499 - val_loss: 4.3499 - val_pitch_loss: 3.3922 - val_duration_loss: 0.9577\n",
      "Epoch 10/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 3.1695 - pitch_loss: 2.9410 - duration_loss: 0.2285 - val_loss: 4.4430 - val_pitch_loss: 3.4407 - val_duration_loss: 1.0023\n",
      "Epoch 11/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 3.1380 - pitch_loss: 2.9262 - duration_loss: 0.2118 - val_loss: 4.5204 - val_pitch_loss: 3.5267 - val_duration_loss: 0.9938\n",
      "Epoch 12/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 3.0943 - pitch_loss: 2.8987 - duration_loss: 0.1957 - val_loss: 4.5928 - val_pitch_loss: 3.5403 - val_duration_loss: 1.0524\n",
      "Epoch 13/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 3.0397 - pitch_loss: 2.8557 - duration_loss: 0.1840 - val_loss: 4.6402 - val_pitch_loss: 3.5536 - val_duration_loss: 1.0866\n",
      "Epoch 14/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.9903 - pitch_loss: 2.8175 - duration_loss: 0.1729 - val_loss: 4.6841 - val_pitch_loss: 3.6022 - val_duration_loss: 1.0819\n",
      "Epoch 15/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.9434 - pitch_loss: 2.7807 - duration_loss: 0.1627 - val_loss: 4.7599 - val_pitch_loss: 3.6374 - val_duration_loss: 1.1225\n",
      "Epoch 16/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.8996 - pitch_loss: 2.7444 - duration_loss: 0.1552 - val_loss: 4.7667 - val_pitch_loss: 3.6250 - val_duration_loss: 1.1417\n",
      "Epoch 17/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.8586 - pitch_loss: 2.7122 - duration_loss: 0.1465 - val_loss: 4.8680 - val_pitch_loss: 3.6971 - val_duration_loss: 1.1709\n",
      "Epoch 18/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.8208 - pitch_loss: 2.6808 - duration_loss: 0.1400 - val_loss: 5.0671 - val_pitch_loss: 3.7963 - val_duration_loss: 1.2707\n",
      "Epoch 19/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.7720 - pitch_loss: 2.6403 - duration_loss: 0.1317 - val_loss: 5.0250 - val_pitch_loss: 3.7834 - val_duration_loss: 1.2417\n",
      "Epoch 20/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.7334 - pitch_loss: 2.6058 - duration_loss: 0.1276 - val_loss: 5.1090 - val_pitch_loss: 3.8317 - val_duration_loss: 1.2773\n",
      "Epoch 21/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.7056 - pitch_loss: 2.5822 - duration_loss: 0.1235 - val_loss: 5.1553 - val_pitch_loss: 3.8532 - val_duration_loss: 1.3020\n",
      "Epoch 22/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.6728 - pitch_loss: 2.5527 - duration_loss: 0.1201 - val_loss: 5.2329 - val_pitch_loss: 3.8891 - val_duration_loss: 1.3437\n",
      "Epoch 23/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.6422 - pitch_loss: 2.5261 - duration_loss: 0.1161 - val_loss: 5.2139 - val_pitch_loss: 3.8899 - val_duration_loss: 1.3240\n",
      "Epoch 24/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.6072 - pitch_loss: 2.4943 - duration_loss: 0.1129 - val_loss: 5.2547 - val_pitch_loss: 3.9340 - val_duration_loss: 1.3206\n",
      "Epoch 25/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.5827 - pitch_loss: 2.4716 - duration_loss: 0.1110 - val_loss: 5.3953 - val_pitch_loss: 4.0153 - val_duration_loss: 1.3799\n",
      "Epoch 26/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.5555 - pitch_loss: 2.4481 - duration_loss: 0.1074 - val_loss: 5.4249 - val_pitch_loss: 4.0788 - val_duration_loss: 1.3461\n",
      "Epoch 27/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.5290 - pitch_loss: 2.4238 - duration_loss: 0.1052 - val_loss: 5.4660 - val_pitch_loss: 4.0950 - val_duration_loss: 1.3710\n",
      "Epoch 28/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.5085 - pitch_loss: 2.4056 - duration_loss: 0.1029 - val_loss: 5.3659 - val_pitch_loss: 4.0444 - val_duration_loss: 1.3215\n",
      "Epoch 29/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.4805 - pitch_loss: 2.3791 - duration_loss: 0.1014 - val_loss: 5.5121 - val_pitch_loss: 4.1191 - val_duration_loss: 1.3929\n",
      "Epoch 30/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.4603 - pitch_loss: 2.3603 - duration_loss: 0.1001 - val_loss: 5.5645 - val_pitch_loss: 4.1618 - val_duration_loss: 1.4027\n",
      "Epoch 31/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.4308 - pitch_loss: 2.3331 - duration_loss: 0.0977 - val_loss: 5.5904 - val_pitch_loss: 4.2178 - val_duration_loss: 1.3726\n",
      "Epoch 32/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.4073 - pitch_loss: 2.3117 - duration_loss: 0.0956 - val_loss: 5.6686 - val_pitch_loss: 4.2203 - val_duration_loss: 1.4483\n",
      "Epoch 33/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.3850 - pitch_loss: 2.2913 - duration_loss: 0.0937 - val_loss: 5.6470 - val_pitch_loss: 4.2483 - val_duration_loss: 1.3987\n",
      "Epoch 34/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.3664 - pitch_loss: 2.2733 - duration_loss: 0.0931 - val_loss: 5.7690 - val_pitch_loss: 4.3397 - val_duration_loss: 1.4293\n",
      "Epoch 35/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.3480 - pitch_loss: 2.2569 - duration_loss: 0.0911 - val_loss: 5.8922 - val_pitch_loss: 4.3782 - val_duration_loss: 1.5141\n",
      "Epoch 36/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.3282 - pitch_loss: 2.2365 - duration_loss: 0.0916 - val_loss: 5.8951 - val_pitch_loss: 4.3952 - val_duration_loss: 1.4999\n",
      "Epoch 37/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.3125 - pitch_loss: 2.2219 - duration_loss: 0.0906 - val_loss: 5.8491 - val_pitch_loss: 4.4102 - val_duration_loss: 1.4389\n",
      "Epoch 38/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.2918 - pitch_loss: 2.2011 - duration_loss: 0.0907 - val_loss: 5.8661 - val_pitch_loss: 4.3967 - val_duration_loss: 1.4694\n",
      "Epoch 39/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.2747 - pitch_loss: 2.1854 - duration_loss: 0.0894 - val_loss: 5.9598 - val_pitch_loss: 4.5063 - val_duration_loss: 1.4535\n",
      "Epoch 40/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.2641 - pitch_loss: 2.1755 - duration_loss: 0.0886 - val_loss: 5.9983 - val_pitch_loss: 4.5089 - val_duration_loss: 1.4895\n",
      "Epoch 41/2000000\n",
      "1786/1786 [==============================] - 24s 13ms/step - loss: 2.2470 - pitch_loss: 2.1591 - duration_loss: 0.0879 - val_loss: 6.0751 - val_pitch_loss: 4.5806 - val_duration_loss: 1.4945\n",
      "Epoch 42/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.2325 - pitch_loss: 2.1467 - duration_loss: 0.0857 - val_loss: 5.9596 - val_pitch_loss: 4.4657 - val_duration_loss: 1.4939\n",
      "Epoch 43/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.2195 - pitch_loss: 2.1327 - duration_loss: 0.0868 - val_loss: 5.9985 - val_pitch_loss: 4.5255 - val_duration_loss: 1.4730\n",
      "Epoch 44/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.2047 - pitch_loss: 2.1206 - duration_loss: 0.0841 - val_loss: 6.0751 - val_pitch_loss: 4.5678 - val_duration_loss: 1.5072\n",
      "Epoch 45/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.1864 - pitch_loss: 2.1024 - duration_loss: 0.0840 - val_loss: 6.0801 - val_pitch_loss: 4.5851 - val_duration_loss: 1.4950\n",
      "Epoch 46/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.1841 - pitch_loss: 2.0999 - duration_loss: 0.0842 - val_loss: 6.3306 - val_pitch_loss: 4.7161 - val_duration_loss: 1.6145\n",
      "Epoch 47/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.1697 - pitch_loss: 2.0844 - duration_loss: 0.0854 - val_loss: 6.1921 - val_pitch_loss: 4.6606 - val_duration_loss: 1.5315\n",
      "Epoch 48/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.1609 - pitch_loss: 2.0773 - duration_loss: 0.0836 - val_loss: 6.1444 - val_pitch_loss: 4.6215 - val_duration_loss: 1.5229\n",
      "Epoch 49/2000000\n",
      "1786/1786 [==============================] - 24s 14ms/step - loss: 2.1482 - pitch_loss: 2.0664 - duration_loss: 0.0818 - val_loss: 6.2645 - val_pitch_loss: 4.6689 - val_duration_loss: 1.5956\n",
      "Epoch 50/2000000\n",
      "1786/1786 [==============================] - 25s 14ms/step - loss: 2.1464 - pitch_loss: 2.0631 - duration_loss: 0.0833 - val_loss: 6.3053 - val_pitch_loss: 4.7229 - val_duration_loss: 1.5824\n",
      "Epoch 51/2000000\n",
      "1786/1786 [==============================] - 44s 24ms/step - loss: 2.1299 - pitch_loss: 2.0474 - duration_loss: 0.0824 - val_loss: 6.3616 - val_pitch_loss: 4.7767 - val_duration_loss: 1.5848\n",
      "Epoch 52/2000000\n",
      "1786/1786 [==============================] - 55s 30ms/step - loss: 2.1207 - pitch_loss: 2.0393 - duration_loss: 0.0814 - val_loss: 6.2610 - val_pitch_loss: 4.7197 - val_duration_loss: 1.5413\n",
      "Epoch 53/2000000\n",
      "1049/1786 [================>.............] - ETA: 12s - loss: 2.0968 - pitch_loss: 2.0187 - duration_loss: 0.0781"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[14], line 33\u001b[0m\n\u001b[1;32m     26\u001b[0m callbacks_list \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     27\u001b[0m     checkpoint1\n\u001b[1;32m     28\u001b[0m     , checkpoint2\n\u001b[1;32m     29\u001b[0m     , early_stopping\n\u001b[1;32m     30\u001b[0m  ]\n\u001b[1;32m     32\u001b[0m model\u001b[38;5;241m.\u001b[39msave_weights(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(weights_folder, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mweights.h5\u001b[39m\u001b[38;5;124m\"\u001b[39m))\n\u001b[0;32m---> 33\u001b[0m \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfit\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnetwork_input\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnetwork_output\u001b[49m\n\u001b[1;32m     34\u001b[0m \u001b[43m          \u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mepochs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m2000000\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;241;43m64\u001b[39;49m\n\u001b[1;32m     35\u001b[0m \u001b[43m          \u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalidation_split\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;241;43m0.2\u001b[39;49m\n\u001b[1;32m     36\u001b[0m \u001b[43m          \u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcallbacks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallbacks_list\u001b[49m\n\u001b[1;32m     37\u001b[0m \u001b[43m          \u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshuffle\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\n\u001b[1;32m     38\u001b[0m \u001b[43m         \u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/keras/utils/traceback_utils.py:65\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     63\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 65\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     66\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     67\u001b[0m     filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/keras/engine/training.py:1650\u001b[0m, in \u001b[0;36mModel.fit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1642\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m tf\u001b[38;5;241m.\u001b[39mprofiler\u001b[38;5;241m.\u001b[39mexperimental\u001b[38;5;241m.\u001b[39mTrace(\n\u001b[1;32m   1643\u001b[0m     \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtrain\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m   1644\u001b[0m     epoch_num\u001b[38;5;241m=\u001b[39mepoch,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1647\u001b[0m     _r\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m,\n\u001b[1;32m   1648\u001b[0m ):\n\u001b[1;32m   1649\u001b[0m     callbacks\u001b[38;5;241m.\u001b[39mon_train_batch_begin(step)\n\u001b[0;32m-> 1650\u001b[0m     tmp_logs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_function\u001b[49m\u001b[43m(\u001b[49m\u001b[43miterator\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1651\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m data_handler\u001b[38;5;241m.\u001b[39mshould_sync:\n\u001b[1;32m   1652\u001b[0m         context\u001b[38;5;241m.\u001b[39masync_wait()\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/tensorflow/python/util/traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:880\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    877\u001b[0m compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    879\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m OptionalXlaContext(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile):\n\u001b[0;32m--> 880\u001b[0m   result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    882\u001b[0m new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n\u001b[1;32m    883\u001b[0m without_tracing \u001b[38;5;241m=\u001b[39m (tracing_count \u001b[38;5;241m==\u001b[39m new_tracing_count)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:912\u001b[0m, in \u001b[0;36mFunction._call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    909\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n\u001b[1;32m    910\u001b[0m   \u001b[38;5;66;03m# In this case we have created variables on the first call, so we run the\u001b[39;00m\n\u001b[1;32m    911\u001b[0m   \u001b[38;5;66;03m# defunned version which is guaranteed to never create variables.\u001b[39;00m\n\u001b[0;32m--> 912\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_no_variable_creation_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m  \u001b[38;5;66;03m# pylint: disable=not-callable\u001b[39;00m\n\u001b[1;32m    913\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_variable_creation_fn \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    914\u001b[0m   \u001b[38;5;66;03m# Release the lock early so that multiple threads can perform the call\u001b[39;00m\n\u001b[1;32m    915\u001b[0m   \u001b[38;5;66;03m# in parallel.\u001b[39;00m\n\u001b[1;32m    916\u001b[0m   \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock\u001b[38;5;241m.\u001b[39mrelease()\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/tensorflow/python/eager/polymorphic_function/tracing_compiler.py:134\u001b[0m, in \u001b[0;36mTracingCompiler.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    131\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_lock:\n\u001b[1;32m    132\u001b[0m   (concrete_function,\n\u001b[1;32m    133\u001b[0m    filtered_flat_args) \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_maybe_define_function(args, kwargs)\n\u001b[0;32m--> 134\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mconcrete_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_flat\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    135\u001b[0m \u001b[43m    \u001b[49m\u001b[43mfiltered_flat_args\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptured_inputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mconcrete_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcaptured_inputs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py:1745\u001b[0m, in \u001b[0;36mConcreteFunction._call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1741\u001b[0m possible_gradient_type \u001b[38;5;241m=\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPossibleTapeGradientTypes(args)\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (possible_gradient_type \u001b[38;5;241m==\u001b[39m gradients_util\u001b[38;5;241m.\u001b[39mPOSSIBLE_GRADIENT_TYPES_NONE\n\u001b[1;32m   1743\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m executing_eagerly):\n\u001b[1;32m   1744\u001b[0m   \u001b[38;5;66;03m# No tape is watching; skip to running the function.\u001b[39;00m\n\u001b[0;32m-> 1745\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_build_call_outputs(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_inference_function\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcall\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1746\u001b[0m \u001b[43m      \u001b[49m\u001b[43mctx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcancellation_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcancellation_manager\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[1;32m   1747\u001b[0m forward_backward \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_select_forward_and_backward_functions(\n\u001b[1;32m   1748\u001b[0m     args,\n\u001b[1;32m   1749\u001b[0m     possible_gradient_type,\n\u001b[1;32m   1750\u001b[0m     executing_eagerly)\n\u001b[1;32m   1751\u001b[0m forward_function, args_with_tangents \u001b[38;5;241m=\u001b[39m forward_backward\u001b[38;5;241m.\u001b[39mforward()\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/tensorflow/python/eager/polymorphic_function/monomorphic_function.py:378\u001b[0m, in \u001b[0;36m_EagerDefinedFunction.call\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    376\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m _InterpolateFunctionError(\u001b[38;5;28mself\u001b[39m):\n\u001b[1;32m    377\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m cancellation_manager \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 378\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m \u001b[43mexecute\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    379\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msignature\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    380\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_num_outputs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    381\u001b[0m \u001b[43m        \u001b[49m\u001b[43minputs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    382\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattrs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    383\u001b[0m \u001b[43m        \u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mctx\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    384\u001b[0m   \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    385\u001b[0m     outputs \u001b[38;5;241m=\u001b[39m execute\u001b[38;5;241m.\u001b[39mexecute_with_cancellation(\n\u001b[1;32m    386\u001b[0m         \u001b[38;5;28mstr\u001b[39m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msignature\u001b[38;5;241m.\u001b[39mname),\n\u001b[1;32m    387\u001b[0m         num_outputs\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_num_outputs,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    390\u001b[0m         ctx\u001b[38;5;241m=\u001b[39mctx,\n\u001b[1;32m    391\u001b[0m         cancellation_manager\u001b[38;5;241m=\u001b[39mcancellation_manager)\n",
      "File \u001b[0;32m/opt/conda/lib/python3.10/site-packages/tensorflow/python/eager/execute.py:52\u001b[0m, in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     51\u001b[0m   ctx\u001b[38;5;241m.\u001b[39mensure_initialized()\n\u001b[0;32m---> 52\u001b[0m   tensors \u001b[38;5;241m=\u001b[39m \u001b[43mpywrap_tfe\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mTFE_Py_Execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mctx\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_handle\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdevice_name\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mop_name\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     53\u001b[0m \u001b[43m                                      \u001b[49m\u001b[43minputs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattrs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mnum_outputs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     54\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m core\u001b[38;5;241m.\u001b[39m_NotOkStatusException \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m     55\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m name \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "weights_folder = os.path.join(run_folder, 'weights')\n",
    "\n",
    "checkpoint1 = ModelCheckpoint(\n",
    "    os.path.join(weights_folder, \"weights-improvement-{epoch:02d}-{loss:.4f}-bigger.h5\"),\n",
    "    monitor='loss',\n",
    "    verbose=0,\n",
    "    save_best_only=True,\n",
    "    mode='min'\n",
    ")\n",
    "\n",
    "checkpoint2 = ModelCheckpoint(\n",
    "    os.path.join(weights_folder, \"weights.h5\"),\n",
    "    monitor='loss',\n",
    "    verbose=0,\n",
    "    save_best_only=True,\n",
    "    mode='min'\n",
    ")\n",
    "\n",
    "early_stopping = EarlyStopping(\n",
    "    monitor='loss'\n",
    "    , restore_best_weights=True\n",
    "    , patience = 10\n",
    ")\n",
    "\n",
    "\n",
    "callbacks_list = [\n",
    "    checkpoint1\n",
    "    , checkpoint2\n",
    "    , early_stopping\n",
    " ]\n",
    "\n",
    "model.save_weights(os.path.join(weights_folder, \"weights.h5\"))\n",
    "model.fit(network_input, network_output\n",
    "          , epochs=2000000, batch_size=64\n",
    "          , validation_split = 0.2\n",
    "          , callbacks=callbacks_list\n",
    "          , shuffle=True\n",
    "         )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}