status.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Reporting status of the simulation and doing automatic analysis if run by cron\n",
    "## NOTE: ALWAYS comment out calls to submitting a job if you enable cron running!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "\n",
       "# MD simulations of H3-H4 with 30 bp of DNA, tails truncated\n",
       "- AMBER14SB force field\n",
       "- 150 mM NaCl\n",
       "- box 2nm\n"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Folder set to:  h3-h4_tm_30DNA\n",
      "Project name:  h3-h4_tm_30DNA\n",
      "SSH host set to: lomo2g\n"
     ]
    }
   ],
   "source": [
    "#Reloading funcs if changed\n",
    "from importlib import reload \n",
    "import funcs,config\n",
    "reload(funcs)\n",
    "reload(config)\n",
    "\n",
    "\n",
    "from config import movtitle\n",
    "from IPython.display import Markdown, display\n",
    "from config import folder,descr,pname,ssh_host\n",
    "display(Markdown(descr))\n",
    "print(\"Folder set to: \",folder)\n",
    "print(\"Project name: \",pname)\n",
    "print(\"SSH host set to:\",ssh_host)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!./get_files.sh 6\n",
    "#!gacp\n",
    "#!ssh $ssh_host \"mkdir -p ~/_scratch/$folder\" \n",
    "#!ssh $ssh_host \"mkdir -p ~/_scratch/$folder/GMX_run\" \n",
    "#!rsync --partial GMX_run/7_prod* $ssh_host:~/_scratch/$folder/GMX_run/;\n",
    "#!rsync --partial GMX_run/lomo2* $ssh_host:~/_scratch/$folder/GMX_run/;    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!ssh $ssh_host \"module load slurm gromacs/2018-gcc; squeue -n $folder \" || echo \"No job in queue\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           Step           Time\r\n",
      "      561106000   1122212.00000\r\n",
      "--\r\n",
      "           Step           Time\r\n",
      "      561107000   1122214.00000\r\n",
      "--\r\n",
      "           Step           Time\r\n",
      "      561108000   1122216.00000\r\n"
     ]
    }
   ],
   "source": [
    "!tail -n 50 GMX_run/7_prod.log | grep -A1 Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "           Step           Time\n",
      "      561106000   1122212.00000\n",
      "--\n",
      "           Step           Time\n",
      "      561107000   1122214.00000\n",
      "--\n",
      "           Step           Time\n",
      "      561108000   1122216.00000\n"
     ]
    }
   ],
   "source": [
    "!ssh $ssh_host \"tail -n 50 ~/_scratch/$folder/GMX_run/7_prod.log | grep -A1 Time\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "-rw-rw-r-- 1 gorkovets_1609 gorkovets_1609 379432324 Apr  6 04:37 /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/7_prod.log\n"
     ]
    }
   ],
   "source": [
    "!ssh $ssh_host \"ls -l ~/_scratch/$folder/GMX_run/7_prod.log \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "1_minim.edr\n",
      "1_minim.gro\n",
      "1_minim.log\n",
      "1_minim.tpr\n",
      "2_equil.cpt\n",
      "2_equil.edr\n",
      "2_equil.gro\n",
      "2_equil.log\n",
      "2_equil_prev.cpt\n",
      "2_equil.tpr\n",
      "3_equil.cpt\n",
      "3_equil.edr\n",
      "3_equil.gro\n",
      "3_equil.log\n",
      "3_equil_prev.cpt\n",
      "3_equil.tpr\n",
      "4_equil.cpt\n",
      "4_equil.edr\n",
      "4_equil.gro\n",
      "4_equil.log\n",
      "4_equil_prev.cpt\n",
      "4_equil.tpr\n",
      "5_equil.cpt\n",
      "5_equil.edr\n",
      "5_equil.gro\n",
      "5_equil.log\n",
      "5_equil_prev.cpt\n",
      "5_equil.tpr\n",
      "6_equil.cpt\n",
      "6_equil.edr\n",
      "6_equil.gro\n",
      "6_equil.log\n",
      "6_equil.pdb\n",
      "6_equil_prev.cpt\n",
      "6_equil.tpr\n",
      "7_prod.tpr\n",
      "egmx.1034691\n",
      "lomo2_run_cont.sh\n",
      "lomo2_run.sh\n",
      "mdout_1_minim.mdp\n",
      "mdout_2_equil.mdp\n",
      "mdout_3_equil.mdp\n",
      "mdout_4_equil.mdp\n",
      "mdout_5_equil.mdp\n",
      "mdout_6_equil.mdp\n",
      "mdout_7_prod.mdp\n",
      "ogmx.1034691\n",
      "step11b.pdb\n",
      "step11c.pdb\n",
      "step12b.pdb\n",
      "step12c.pdb\n",
      "step15b.pdb\n",
      "step15c.pdb\n",
      "step16b.pdb\n",
      "step16c.pdb\n",
      "step18b.pdb\n",
      "step18c.pdb\n",
      "step19b.pdb\n",
      "step19c.pdb\n",
      "step21b.pdb\n",
      "step21c.pdb\n",
      "step27b.pdb\n",
      "step27c.pdb\n",
      "step31b.pdb\n",
      "step31c.pdb\n",
      "step390b.pdb\n",
      "step390c.pdb\n"
     ]
    }
   ],
   "source": [
    "!ssh $ssh_host \"ls ~/_scratch/$folder/GMX_run/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939953 <==\n",
      "\n",
      "NOTE: GROMACS was configured without NVML support hence it can not exploit\n",
      "      application clocks of the detected Tesla K40st GPU to improve performance.\n",
      "      Recompile with the NVML library (compatible with the driver used) or set application clocks manually.\n",
      "\n",
      "Non-default thread affinity set probably by the OpenMP library,\n",
      "disabling internal thread affinity\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 0.8%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 0.4%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    10933.561      911.130     1200.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:        9.483        2.531\n",
      "\n",
      "GROMACS reminds you: \"Life in the streets is not easy\" (Marky Mark)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939954 <==\n",
      "      application clocks of the detected Tesla K40s GPU to improve performance.\n",
      "      Recompile with the NVML library (compatible with the driver used) or set application clocks manually.\n",
      "\n",
      "NOTE: The number of threads is not equal to the number of (logical) cores\n",
      "      and the -pin option is set to auto: will not pin threads to cores.\n",
      "      This can lead to significant performance degradation.\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 2.6%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.3%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:     5092.445      212.185     2400.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       40.720        0.589\n",
      "\n",
      "GROMACS reminds you: \"Your Bones Got a Little Machine\" (Pixies)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939955 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1900 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.2 %.\n",
      "\n",
      "\n",
      "step 4000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 2.8%.\n",
      " The balanceable part of the MD step is 49%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.4%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:     5878.238      163.284     3600.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       52.915        0.454\n",
      "\n",
      "GROMACS reminds you: \"What about my nose?\" (Aneesur Rahman, responding to an Argonne manager arguing the long hair of Charles Bennett in his group was disreputing the lab; Retold by Michael Klein)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939956 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.6 %.\n",
      "\n",
      "\n",
      "step 17000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 2.0%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.0%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:     6607.111      137.648     4800.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       62.770        0.382\n",
      "\n",
      "GROMACS reminds you: \"A Pretty Village Burning Makes a Pretty Fire\" (David Sandstrom)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939957 <==\n",
      "      application clocks of the detected Tesla K40s GPU to improve performance.\n",
      "      Recompile with the NVML library (compatible with the driver used) or set application clocks manually.\n",
      "\n",
      "NOTE: The number of threads is not equal to the number of (logical) cores\n",
      "      and the -pin option is set to auto: will not pin threads to cores.\n",
      "      This can lead to significant performance degradation.\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 2.1%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.0%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:     8050.088      134.168     6000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       64.398        0.373\n",
      "\n",
      "GROMACS reminds you: \"Sacrifices must be made\" (Otto Lilienthal, dying after having crashed with his glider in 1896)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939958 <==\n",
      "      application clocks of the detected Tesla K40s GPU to improve performance.\n",
      "      Recompile with the NVML library (compatible with the driver used) or set application clocks manually.\n",
      "\n",
      "NOTE: The number of threads is not equal to the number of (logical) cores\n",
      "      and the -pin option is set to auto: will not pin threads to cores.\n",
      "      This can lead to significant performance degradation.\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 2.9%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.4%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:     9082.729      126.149     7200.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       68.492        0.350\n",
      "\n",
      "GROMACS reminds you: \"The determined Real Programmer can write FORTRAN programs in any language.\" (Ed Post)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939970 <==\n",
      "Working dir:  /mnt/scratch/users/gorkovets_1609/h3-h4_tm_DNA/GMX_run\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench7\n",
      "\n",
      "Reading file bench7.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec_setup.cpp (line 781)\n",
      "MPI rank:    0 (out of 14)\n",
      "\n",
      "Fatal error:\n",
      "The number of ranks you selected (14) contains a large prime factor 7. In most\n",
      "cases this will lead to bad performance. Choose a number with smaller prime\n",
      "factors or set the decomposition (option -dd) manually.\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD\n",
      "with errorcode 1.\n",
      "\n",
      "NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.\n",
      "You may or may not see output from other processes, depending on\n",
      "exactly when Open MPI kills them.\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939971 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1300 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.4 %.\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 2.8%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.4%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    10075.979      104.958     9600.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       82.320        0.292\n",
      "\n",
      "GROMACS reminds you: \"If mathematical analysis should ever hold a prominent place in chemistry - an aberration which is happily almost impossible - it would occasion a rapid and widespread degeneration of that science.\" (Aguste Comte, 1830)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939972 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1200 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.9 %.\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.5%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.8%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    11224.764      103.933    10800.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       83.132        0.289\n",
      "\n",
      "GROMACS reminds you: \"The Carpenter Goes Bang Bang\" (The Breeders)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939973 <==\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 10000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.2 %.\n",
      "\n",
      "\n",
      "step 11000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 4.9%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.4%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    11910.979       99.258    12000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       87.047        0.276\n",
      "\n",
      "GROMACS reminds you: \"Strength is just an accident arising from the weakness of others\" (Joseph Conrad)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939974 <==\n",
      "Working dir:  /mnt/scratch/users/gorkovets_1609/h3-h4_tm_DNA/GMX_run\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench11\n",
      "\n",
      "Reading file bench11.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec_setup.cpp (line 781)\n",
      "MPI rank:    0 (out of 22)\n",
      "\n",
      "Fatal error:\n",
      "The number of ranks you selected (22) contains a large prime factor 11. In\n",
      "most cases this will lead to bad performance. Choose a number with smaller\n",
      "prime factors or set the decomposition (option -dd) manually.\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD\n",
      "with errorcode 1.\n",
      "\n",
      "NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.\n",
      "You may or may not see output from other processes, depending on\n",
      "exactly when Open MPI kills them.\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939975 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1300 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.3 %.\n",
      "\n",
      "\n",
      "step 4000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.4%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.7%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    12675.509       88.024    14400.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       98.157        0.245\n",
      "\n",
      "GROMACS reminds you: \"Problems worthy / of attack / prove their worth / by hitting back.\" (Piet Hein)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.939990 <==\n",
      "\n",
      "NOTE: GROMACS was configured without NVML support hence it can not exploit\n",
      "      application clocks of the detected Tesla K40st GPU to improve performance.\n",
      "      Recompile with the NVML library (compatible with the driver used) or set application clocks manually.\n",
      "\n",
      "NOTE: The number of threads is not equal to the number of (logical) cores\n",
      "      and the -pin option is set to auto: will not pin threads to cores.\n",
      "      This can lead to significant performance degradation.\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "WARNING: This run will generate roughly 3140 Mb of data\n",
      "\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "5000000000 steps, 10000000.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.7 %.\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "mlx5: n53318.10p.parallel.ru: got completion with error:\n",
      "00000000 00000000 00000000 00000000\n",
      "00000000 00000000 00000000 00000000\n",
      "00000000 00000000 00000000 00000000\n",
      "00000000 00008813 0809606d 2c1cb2d1\n",
      "[[31785,1],2][btl_openib_component.c:3556:handle_wc] from n53318 to: n53317 error polling LP CQ with status REMOTE ACCESS ERROR status number 10 for wr_id 8b0580 opcode 1  vendor error 136 qp_idx 0\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940106 <==\n",
      "Working dir:  /mnt/scratch/users/gorkovets_1609/h3-h4_tm_DNA/GMX_run\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench13\n",
      "\n",
      "Reading file bench13.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec_setup.cpp (line 781)\n",
      "MPI rank:    0 (out of 26)\n",
      "\n",
      "Fatal error:\n",
      "The number of ranks you selected (26) contains a large prime factor 13. In\n",
      "most cases this will lead to bad performance. Choose a number with smaller\n",
      "prime factors or set the decomposition (option -dd) manually.\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD\n",
      "with errorcode 1.\n",
      "\n",
      "NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.\n",
      "You may or may not see output from other processes, depending on\n",
      "exactly when Open MPI kills them.\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940107 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 900 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.5 %.\n",
      "\n",
      "\n",
      "step 4000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.0%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.5%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    13983.645       83.236    16800.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      103.803        0.231\n",
      "\n",
      "GROMACS reminds you: \"Fresh Air, Green Hair\" (Frank Black)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940109 <==\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 10000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.2 %.\n",
      "\n",
      "\n",
      "step 11000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 5.4%.\n",
      " The balanceable part of the MD step is 49%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.7%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    15808.895       87.827    18000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:       98.377        0.244\n",
      "\n",
      "GROMACS reminds you: \"Sort Of\" (Urban Dance Squad)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940110 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 900 Turning on dynamic load balancing, because the performance loss due to load imbalance is 4.2 %.\n",
      "\n",
      "\n",
      "step 4000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.1%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.6%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    14619.279       76.142    19200.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      113.474        0.212\n",
      "\n",
      "GROMACS reminds you: \"Welcome to the Power Age\" (2 Unlimited)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940111 <==\n",
      "Working dir:  /mnt/scratch/users/gorkovets_1609/h3-h4_tm_DNA/GMX_run\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench17\n",
      "\n",
      "Reading file bench17.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec_setup.cpp (line 781)\n",
      "MPI rank:    0 (out of 34)\n",
      "\n",
      "Fatal error:\n",
      "The number of ranks you selected (34) contains a large prime factor 17. In\n",
      "most cases this will lead to bad performance. Choose a number with smaller\n",
      "prime factors or set the decomposition (option -dd) manually.\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD\n",
      "with errorcode 1.\n",
      "\n",
      "NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.\n",
      "You may or may not see output from other processes, depending on\n",
      "exactly when Open MPI kills them.\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940181 <==\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940182 <==\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940183 <==\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940184 <==\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940185 <==\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940521 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1600 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.9 %.\n",
      "\n",
      "\n",
      "step 4000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.1%.\n",
      " The balanceable part of the MD step is 49%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.5%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    16440.948       76.116    21600.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      113.514        0.211\n",
      "\n",
      "GROMACS reminds you: \"Sir, spare your threats: The bug which you would fright me with I seek.\" (Hermione, Act III, scene II of Shakespeare's Winter's Tale)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940522 <==\n",
      "Working dir:  /mnt/scratch/users/gorkovets_1609/h3-h4_tm_DNA/GMX_run\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench19\n",
      "\n",
      "Reading file bench19.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec_setup.cpp (line 781)\n",
      "MPI rank:    0 (out of 38)\n",
      "\n",
      "Fatal error:\n",
      "The number of ranks you selected (38) contains a large prime factor 19. In\n",
      "most cases this will lead to bad performance. Choose a number with smaller\n",
      "prime factors or set the decomposition (option -dd) manually.\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD\n",
      "with errorcode 1.\n",
      "\n",
      "NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.\n",
      "You may or may not see output from other processes, depending on\n",
      "exactly when Open MPI kills them.\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940523 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 700 Turning on dynamic load balancing, because the performance loss due to load imbalance is 4.4 %.\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.4%.\n",
      " The balanceable part of the MD step is 49%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.7%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    16820.809       70.087    24000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      123.278        0.195\n",
      "\n",
      "GROMACS reminds you: \"You Can Always Go On Ricky Lake\" (Offspring)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940524 <==\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1600 Turning on dynamic load balancing, because the performance loss due to load imbalance is 5.1 %.\n",
      "\n",
      "\n",
      "step 4000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 5000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.0 %.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was turned on during the run due to measured imbalance.\n",
      " Average load imbalance: 1.9%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.0%.\n",
      " Steps where the load balancing was limited by -rdd, -rcon and/or -dds: X 0 % Y 1 %\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    18467.420       73.283    25200.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      117.901        0.204\n",
      "\n",
      "GROMACS reminds you: \"There are two major products that come out of Berkeley: LSD and UNIX. We don't believe this to be a coincidence.\" (Jeremy Anderson)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940525 <==\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench22\n",
      "\n",
      "Reading file bench22.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec.cpp (line 6571)\n",
      "MPI rank:    0 (out of 44)\n",
      "\n",
      "Fatal error:\n",
      "There is no domain decomposition for 44 ranks that is compatible with the\n",
      "given box and a minimum cell size of 1.02425 nm\n",
      "Change the number of ranks or mdrun option -rcon or -dds or your LINCS\n",
      "settings\n",
      "Look in the log file for details on the domain decomposition\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "mpirun detected that one or more processes exited with non-zero status, thus causing\n",
      "the job to be terminated. The first process to do so was:\n",
      "\n",
      "  Process name: [[3126,1],39]\n",
      "  Exit code:    1\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940531 <==\n",
      "      and the -pin option is set to auto: will not pin threads to cores.\n",
      "      This can lead to significant performance degradation.\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1300 Turning on dynamic load balancing, because the performance loss due to load imbalance is 4.8 %.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was turned on during the run due to measured imbalance.\n",
      " Average load imbalance: 2.1%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.1%.\n",
      " Steps where the load balancing was limited by -rdd, -rcon and/or -dds: X 0 % Y 1 %\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    20093.780       69.770    28800.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      123.838        0.194\n",
      "\n",
      "GROMACS reminds you: \"People are DNA's way of making more DNA.\" (Edward O. Wilson)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940532 <==\n",
      "\n",
      "\n",
      "step 4000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 10000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.2 %.\n",
      "\n",
      "\n",
      "step 12000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 4.4%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.2%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    21597.196       71.991    30000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      120.018        0.200\n",
      "\n",
      "GROMACS reminds you: \"We'll celebrate a woman for anything, as long as it's not her talent.\" (Colleen McCullough)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940533 <==\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench26\n",
      "\n",
      "Reading file bench26.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec.cpp (line 6571)\n",
      "MPI rank:    0 (out of 52)\n",
      "\n",
      "Fatal error:\n",
      "There is no domain decomposition for 52 ranks that is compatible with the\n",
      "given box and a minimum cell size of 1.02425 nm\n",
      "Change the number of ranks or mdrun option -rcon or -dds or your LINCS\n",
      "settings\n",
      "Look in the log file for details on the domain decomposition\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "mpirun detected that one or more processes exited with non-zero status, thus causing\n",
      "the job to be terminated. The first process to do so was:\n",
      "\n",
      "  Process name: [[2119,1],39]\n",
      "  Exit code:    1\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940534 <==\n",
      "      and the -pin option is set to auto: will not pin threads to cores.\n",
      "      This can lead to significant performance degradation.\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 700 Turning on dynamic load balancing, because the performance loss due to load imbalance is 4.9 %.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was turned on during the run due to measured imbalance.\n",
      " Average load imbalance: 2.2%.\n",
      " The balanceable part of the MD step is 51%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.1%.\n",
      " Steps where the load balancing was limited by -rdd, -rcon and/or -dds: X 1 % Y 1 %\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    21425.917       66.129    32400.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      130.656        0.184\n",
      "\n",
      "GROMACS reminds you: \"It Just Tastes Better\" (Burger King)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940535 <==\n",
      "\n",
      "\n",
      "step 2000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 10000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.0 %.\n",
      "\n",
      "\n",
      "step 11000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 4.5%.\n",
      " The balanceable part of the MD step is 48%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.2%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    21971.513       65.391    33600.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      132.130        0.182\n",
      "\n",
      "GROMACS reminds you: \"Let's Unzip And Let's Unfold\" (Red Hot Chili Peppers)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940539 <==\n",
      "Working dir:  /mnt/scratch/users/gorkovets_1609/h3-h4_tm_DNA/GMX_run\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench29\n",
      "\n",
      "Reading file bench29.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec_setup.cpp (line 781)\n",
      "MPI rank:    0 (out of 58)\n",
      "\n",
      "Fatal error:\n",
      "The number of ranks you selected (58) contains a large prime factor 29. In\n",
      "most cases this will lead to bad performance. Choose a number with smaller\n",
      "prime factors or set the decomposition (option -dd) manually.\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD\n",
      "with errorcode 1.\n",
      "\n",
      "NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.\n",
      "You may or may not see output from other processes, depending on\n",
      "exactly when Open MPI kills them.\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940540 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1100 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.2 %.\n",
      "\n",
      "\n",
      "step 4000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.5%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.7%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    24956.773       69.324    36000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      124.634        0.193\n",
      "\n",
      "GROMACS reminds you: \"We haven’t the money, so we’ve got to think.\" (Ernest Rutherford)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940541 <==\n",
      "Working dir:  /mnt/scratch/users/gorkovets_1609/h3-h4_tm_DNA/GMX_run\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench31\n",
      "\n",
      "Reading file bench31.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec_setup.cpp (line 781)\n",
      "MPI rank:    0 (out of 62)\n",
      "\n",
      "Fatal error:\n",
      "The number of ranks you selected (62) contains a large prime factor 31. In\n",
      "most cases this will lead to bad performance. Choose a number with smaller\n",
      "prime factors or set the decomposition (option -dd) manually.\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD\n",
      "with errorcode 1.\n",
      "\n",
      "NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.\n",
      "You may or may not see output from other processes, depending on\n",
      "exactly when Open MPI kills them.\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940542 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 2700 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.5 %.\n",
      "\n",
      "\n",
      "step 5000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.4%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.7%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    24414.396       63.579    38400.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      135.896        0.177\n",
      "\n",
      "GROMACS reminds you: \"It Was My Pleasure\" (Pulp Fiction)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940545 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 700 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.6 %.\n",
      "\n",
      "\n",
      "step 21000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 3.4%.\n",
      " The balanceable part of the MD step is 50%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.7%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    28626.052       59.638    48000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      144.878        0.166\n",
      "\n",
      "GROMACS reminds you: \"Clickety Clickety Click\" (System Manager From Hell)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940546 <==\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 700 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.3 %.\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was off during the run due to low measured imbalance.\n",
      " Average load imbalance: 4.0%.\n",
      " The balanceable part of the MD step is 48%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 1.9%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    33547.115       58.242    57600.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      148.351        0.162\n",
      "\n",
      "GROMACS reminds you: \"Don't Follow Me Home\" (Throwing Muses)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940547 <==\n",
      "\n",
      "\n",
      "step 6000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.5 %.\n",
      "\n",
      "\n",
      "step 16000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 20000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 5.1%.\n",
      " The balanceable part of the MD step is 49%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.5%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    36513.726       60.856    60000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      141.977        0.169\n",
      "\n",
      "GROMACS reminds you: \"Got Coffee, Got Donuts, Got Wasted\" (F. Black)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940548 <==\n",
      "\n",
      "\n",
      "step 5000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.2 %.\n",
      "\n",
      "\n",
      "step 16000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 20000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 6.7%.\n",
      " The balanceable part of the MD step is 47%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 3.1%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    41191.420       57.210    72000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      151.025        0.159\n",
      "\n",
      "GROMACS reminds you: \"Can I have everything louder than everything else?\" (Deep Purple)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940549 <==\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 10000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.7 %.\n",
      "\n",
      "\n",
      "step 11000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 6.0%.\n",
      " The balanceable part of the MD step is 47%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.8%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    43611.903       56.786    76800.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      152.152        0.158\n",
      "\n",
      "GROMACS reminds you: \"When doing HPC, don't communica\" (Jim Demmel)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940551 <==\n",
      "\n",
      "\n",
      "step 6000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.4 %.\n",
      "\n",
      "\n",
      "step 18000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 20000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 4.8%.\n",
      " The balanceable part of the MD step is 48%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.3%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    53209.750       63.345    84000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      136.399        0.176\n",
      "\n",
      "GROMACS reminds you: \"Nothing shocks me. I'm a scientist.\" (Harrison Ford as Indiana Jones)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940552 <==\n",
      "\n",
      "\n",
      "step 9000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.4 %.\n",
      "\n",
      "\n",
      "step 16000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 20000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 5.1%.\n",
      " The balanceable part of the MD step is 48%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.5%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    53977.865       56.227    96000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      153.666        0.156\n",
      "\n",
      "GROMACS reminds you: \"Screw a Lightbulb in your Head\" (Gogol Bordello)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940652 <==\n",
      "\n",
      "\n",
      "step 6000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.0 %.\n",
      "\n",
      "\n",
      "step 16000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 20000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 6.9%.\n",
      " The balanceable part of the MD step is 46%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 3.2%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    53569.432       55.113    97200.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      156.773        0.153\n",
      "\n",
      "GROMACS reminds you: \"It seemed a good idea at first\" (Gerrit Groenhof)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940653 <==\n",
      "\n",
      "\n",
      "step 7000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.7 %.\n",
      "\n",
      "\n",
      "step 16000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 20000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 5.9%.\n",
      " The balanceable part of the MD step is 47%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.7%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    56866.113       52.654   108000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      164.094        0.146\n",
      "\n",
      "GROMACS reminds you: \"What about my nose?\" (Aneesur Rahman, responding to an Argonne manager arguing the long hair of Charles Bennett in his group was disreputing the lab; Retold by Michael Klein)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940654 <==\n",
      "50000 steps,    100.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 2800 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.8 %.\n",
      "\n",
      "\n",
      "step 5000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.9 %.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB was turned on during the run due to measured imbalance.\n",
      " Average load imbalance: 4.0%.\n",
      " The balanceable part of the MD step is 49%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.0%.\n",
      " Steps where the load balancing was limited by -rdd, -rcon and/or -dds: X 1 % Y 1 % Z 1 %\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    71398.439       59.499   120000.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      145.216        0.165\n",
      "\n",
      "GROMACS reminds you: \"This work contains many things which are new and interesting. Unfortunately, everything that is new is not interesting, and everything which is interesting, is not new.\" (Lev Landau)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940655 <==\n",
      "\n",
      "\n",
      "step 5000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.2 %.\n",
      "\n",
      "\n",
      "step 19000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 20000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 5.6%.\n",
      " The balanceable part of the MD step is 46%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.6%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:    87056.079       56.677   153600.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      152.445        0.157\n",
      "\n",
      "GROMACS reminds you: \"Lottery: A tax on people who are bad at math.\" (Ambrose Bierce)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940656 <==\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm bench184\n",
      "\n",
      "Reading file bench184.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "Changing nstlist from 10 to 50, rlist from 0.801 to 0.926\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/domdec/domdec.cpp (line 6571)\n",
      "MPI rank:    0 (out of 368)\n",
      "\n",
      "Fatal error:\n",
      "There is no domain decomposition for 368 ranks that is compatible with the\n",
      "given box and a minimum cell size of 1.02425 nm\n",
      "Change the number of ranks or mdrun option -rcon or -dds or your LINCS\n",
      "settings\n",
      "Look in the log file for details on the domain decomposition\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "mpirun detected that one or more processes exited with non-zero status, thus causing\n",
      "the job to be terminated. The first process to do so was:\n",
      "\n",
      "  Process name: [[29038,1],359]\n",
      "  Exit code:    1\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.940748 <==\n",
      "\n",
      "\n",
      "step 6000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 15000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 3.3 %.\n",
      "\n",
      "\n",
      "step 17000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n",
      "\n",
      "step 20000 Will no longer try dynamic load balancing, as it degraded performance.\n",
      "\n",
      "\n",
      "Writing final coordinates.\n",
      "\n",
      " Dynamic load balancing report:\n",
      " DLB got disabled because it was unsuitable to use.\n",
      " Average load imbalance: 6.4%.\n",
      " The balanceable part of the MD step is 43%, load imbalance is computed from this.\n",
      " Part of the total run time spent waiting due to load imbalance: 2.7%.\n",
      "\n",
      "\n",
      "               Core t (s)   Wall t (s)        (%)\n",
      "       Time:   221310.006       72.041   307200.0\n",
      "                 (ns/day)    (hour/ns)\n",
      "Performance:      119.934        0.200\n",
      "\n",
      "GROMACS reminds you: \"Jesus Built My Hotrod\" (Ministry)\n",
      "\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.942078 <==\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.959609 <==\n",
      "Command line:\n",
      "  gmx_mpi mdrun -ntomp 6 -gputasks 00 -pme cpu -nb gpu -deffnm 7_prod -cpi 7_prod.cpt -px 7_prod_pullx.xvg -pf 7_prod_pullf.xvg\n",
      "\n",
      "Reading file 7_prod.tpr, VERSION 2018.3 (single precision)\n",
      "The number of OpenMP threads was set by environment variable OMP_NUM_THREADS to 6 (and the command-line setting agreed with that)\n",
      "\n",
      "Reading checkpoint file 7_prod.cpt generated: Sat Jan 19 06:32:27 2019\n",
      "\n",
      "\n",
      "\n",
      "-------------------------------------------------------\n",
      "Program:     gmx mdrun, version 2018.1\n",
      "Source file: src/gromacs/fileio/checkpoint.cpp (line 2264)\n",
      "MPI rank:    0 (out of 8)\n",
      "\n",
      "Fatal error:\n",
      "Checkpoint file is for a system of 169387 atoms, while the current system\n",
      "consists of 169742 atoms\n",
      "\n",
      "For more information and tips for troubleshooting, please check the GROMACS\n",
      "website at http://www.gromacs.org/Documentation/Errors\n",
      "-------------------------------------------------------\n",
      "--------------------------------------------------------------------------\n",
      "MPI_ABORT was invoked on rank 0 in communicator MPI_COMM_WORLD\n",
      "with errorcode 1.\n",
      "\n",
      "NOTE: invoking MPI_ABORT causes Open MPI to kill all MPI processes.\n",
      "You may or may not see output from other processes, depending on\n",
      "exactly when Open MPI kills them.\n",
      "--------------------------------------------------------------------------\n",
      "\n",
      "==> /home/gorkovets_1609/_scratch/h3-h4_tm_DNA/GMX_run/egmx.968164 <==\n",
      "      application clocks of the detected Tesla K40st GPU to improve performance.\n",
      "      Recompile with the NVML library (compatible with the driver used) or set application clocks manually.\n",
      "\n",
      "NOTE: The number of threads is not equal to the number of (logical) cores\n",
      "      and the -pin option is set to auto: will not pin threads to cores.\n",
      "      This can lead to significant performance degradation.\n",
      "      Consider using -pin on (and -pinoffset in case you run multiple jobs).\n",
      "\n",
      "Back Off! I just backed up 7_prod_pullx.xvg to ./#7_prod_pullx.xvg.1#\n",
      "\n",
      "Back Off! I just backed up 7_prod_pullf.xvg to ./#7_prod_pullf.xvg.1#\n",
      "\n",
      "Back Off! I just backed up 7_prod.xtc to ./#7_prod.xtc.1#\n",
      "\n",
      "Back Off! I just backed up 7_prod.edr to ./#7_prod.edr.1#\n",
      "\n",
      "WARNING: This run will generate roughly 3140 Mb of data\n",
      "\n",
      "\n",
      "NOTE: DLB will not turn on during the first phase of PME tuning\n",
      "starting mdrun 'MDANALYSIS FRAME 0: Created by PDBWriter in water'\n",
      "5000000000 steps, 10000000.0 ps.\n",
      "\n",
      "NOTE: DLB can now turn on, when beneficial\n",
      "\n",
      "step 1000 Turning on dynamic load balancing, because the performance loss due to load imbalance is 2.7 %.\n",
      "\n",
      "\n",
      "step 3000 Turning off dynamic load balancing, because it is degrading performance.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!ssh $ssh_host \"tail -n 30 ~/_scratch/$folder/GMX_run/egmx*\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\n",
      "           1034691   compute 1kx5_ntm shaytan_  R       0:47      4 n[50722-50725]\n"
     ]
    }
   ],
   "source": [
    "!ssh $ssh_host \"module load slurm gromacs/2018-gcc; squeue -n $folder \" || echo \"No job in queue\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\n",
      "           1034691   compute 1kx5_ntm shaytan_  R       1:04      4 n[50722-50725]\n"
     ]
    }
   ],
   "source": [
    "!ssh $ssh_host \"module load slurm gromacs/2018-gcc; squeue -u \\`whoami\\` \" || echo \"No job in queue\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "Submitted batch job 996781\n"
     ]
    }
   ],
   "source": [
    "## Relaunch\n",
    "#!ssh $ssh_host \"module load slurm gromacs/2018-gcc; cd ~/_scratch/$folder/GMX_run; sbatch -p compute -t 7-00:00:00 -N 4 -J $folder lomo2_run_cont.sh 7_prod\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Making chain of tasks!!\n",
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!ssh $ssh_host \"module load slurm gromacs/2018-gcc; cd ~/_scratch/$folder/GMX_run; sbatch -p compute -t 7-00:00:00 -N 4 -J $folder --dependency=afterany:970076 lomo2_run_cont.sh 7_prod\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n",
      "   Lomonosov-2 supercomputer welcomes you!\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!rsync --partial --update --progress $ssh_host:~/_scratch/$folder/GMX_run/7_prod* GMX_run/ ;\n",
    "!rsync --partial --update --progress $ssh_host:~/_scratch/$folder/GMX_run/egmx* GMX_run/ ;    \n",
    "!rsync --partial --update --progress $ssh_host:~/_scratch/$folder/GMX_run/ogmx* GMX_run/ ;    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!jupyter kernelspec list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NbConvertApp] Converting notebook 5_trj_preprocessing.ipynb to notebook\n",
      "[NbConvertApp] Executing notebook with kernel: python3\n",
      "[NbConvertApp] ERROR | Timeout waiting for execute reply (30s).\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/preprocessors/execute.py\", line 324, in _wait_for_reply\n",
      "    msg = self.kc.shell_channel.get_msg(timeout=timeout)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/jupyter_client/blocking/channels.py\", line 57, in get_msg\n",
      "    raise Empty\n",
      "_queue.Empty\n",
      "\n",
      "During handling of the above exception, another exception occurred:\n",
      "\n",
      "Traceback (most recent call last):\n",
      "  File \"/opt/miniconda3/envs/moldyn/bin/jupyter-nbconvert\", line 10, in <module>\n",
      "    sys.exit(main())\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/jupyter_core/application.py\", line 266, in launch_instance\n",
      "    return super(JupyterApp, cls).launch_instance(argv=argv, **kwargs)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n",
      "    app.start()\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/nbconvertapp.py\", line 325, in start\n",
      "    self.convert_notebooks()\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/nbconvertapp.py\", line 493, in convert_notebooks\n",
      "    self.convert_single_notebook(notebook_filename)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/nbconvertapp.py\", line 464, in convert_single_notebook\n",
      "    output, resources = self.export_single_notebook(notebook_filename, resources, input_buffer=input_buffer)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/nbconvertapp.py\", line 393, in export_single_notebook\n",
      "    output, resources = self.exporter.from_filename(notebook_filename, resources=resources)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/exporters/exporter.py\", line 174, in from_filename\n",
      "    return self.from_file(f, resources=resources, **kw)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/exporters/exporter.py\", line 192, in from_file\n",
      "    return self.from_notebook_node(nbformat.read(file_stream, as_version=4), resources=resources, **kw)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/exporters/notebook.py\", line 31, in from_notebook_node\n",
      "    nb_copy, resources = super(NotebookExporter, self).from_notebook_node(nb, resources, **kw)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/exporters/exporter.py\", line 134, in from_notebook_node\n",
      "    nb_copy, resources = self._preprocess(nb_copy, resources)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/exporters/exporter.py\", line 311, in _preprocess\n",
      "    nbc, resc = preprocessor(nbc, resc)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/preprocessors/base.py\", line 47, in __call__\n",
      "    return self.preprocess(nb, resources)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/preprocessors/execute.py\", line 262, in preprocess\n",
      "    nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/preprocessors/base.py\", line 69, in preprocess\n",
      "    nb.cells[index], resources = self.preprocess_cell(cell, resources, index)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/preprocessors/execute.py\", line 280, in preprocess_cell\n",
      "    reply, outputs = self.run_cell(cell, cell_index)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/preprocessors/execute.py\", line 348, in run_cell\n",
      "    exec_reply = self._wait_for_reply(msg_id, cell)\n",
      "  File \"/opt/miniconda3/envs/moldyn/lib/python3.7/site-packages/nbconvert/preprocessors/execute.py\", line 337, in _wait_for_reply\n",
      "    raise exception(\"Cell execution timed out\")\n",
      "TimeoutError: Cell execution timed out\n"
     ]
    }
   ],
   "source": [
    "!jupyter nbconvert --ExecutePreprocessor.kernel_name=python3 --to notebook --execute 5_trj_preprocessing.ipynb --output 5_trj_preprocessing.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NbConvertApp] Converting notebook 6_1_basic_analysis.ipynb to notebook\n",
      "[NbConvertApp] Executing notebook with kernel: python3\n"
     ]
    }
   ],
   "source": [
    "!jupyter nbconvert --ExecutePreprocessor.timeout=1200 --ExecutePreprocessor.kernel_name=python3 --to notebook --execute 6_1_basic_analysis.ipynb --output 6_1_basic_analysis.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:moldyn]",
   "language": "python",
   "name": "conda-env-moldyn-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}