Merge pull request #271 from Neurosim-lab/development

PR from development to master - VERSION 0.7.5

CHANGES.md

@@ -1,3 +1,49 @@
+# Version 0.7.5
+
+- Improved NeuroML conversion support
+
+- Make cfg.checkErrors = True by default, but set to False if using multiple cores or batch
+
+- Added methods to rename netParams dict keys in a nested format (needed for GUI)
+
+- Added analysis.plotSpikeStats() func to plot stats of cell rates, ISI CVs and synchronies 
+
+- Added analysis.calculateRate() func to calculate avg and peak rate of pop subset at time range
+
+- Added analysis.plotRates() func to plot avg and peak rate of different pop subsets at time range
+
+- Added option to include list of pops or cells in 'include' arguments e.g. include=[['E4','E2'], [1,3]]
+
+- Added cfg.compactConnFormat option to replace conns dict format with compact list format 
+
+- Added option to plotConn() and plot2Dnet() to load data from compact format json files 
+
+- Adapted python2 code so conversion to python3 via 2to3 command works straight away
+
+- Added 'instantiate' argument to sim.load function
+
+- Added 'dpi' argument to analysis.plotSpikeHist()
+
+- Replaced init()/h.stdinit() with finitialize() so h.v_init and h.dt get set properly
+
+- Removed call to h.stdrun() but made v_init a global in cfg.hParams and initialized h.tstop to cfg.duration
+
+- Fixed bug setting globals that don't exist
+
+- Fixed issue setting global variables when loading from json
+
+- Fixed issue to make convergence+divergence connections randomization more robust and efficient (issue #254)
+
+- Fixed bug in colors of plotSpikeHist
+
+- Fixed bug in replaceDictODict() that lead to wrong results when importing cells
+
+- Fixed bug when using sim.gatherOnlySimData
+
+- Fixed bugs in saveLoadV1 example
+
+- Fixed bug when generating subConn with createNEURONObj=False
+
 # Version 0.7.4
 
 - Added polarity param to analysis.plotEPSPAmp() 
@@ -14,6 +60,7 @@
 
 - Fixed bug that removed previously existing element from path during importCellParams()
 
+
 # Version 0.7.3
 
 - Option to create section lists based on y displacement from soma (addCellParamsSecList)

doc/source/code/tut8_analysis.py

@@ -0,0 +1,209 @@
+"""
+analysis.py 
+
+Functions to read and plot figures from the batch simulation results. 
+
+Contributors: [email protected]
+"""
+
+import json
+import pandas as pd
+import seaborn as sb
+import matplotlib.pyplot as plt
+import pickle
+import numpy as np
+from pylab import *
+from itertools import product
+from pprint import pprint
+from netpyne import specs
+
+
+#--------------------------------------------------------------------
+# Function to read batch data
+#--------------------------------------------------------------------
+def readBatchData(dataFolder, batchLabel, loadAll=False, saveAll=True, vars=None, maxCombs=None, listCombs=None):
+    # load from previously saved file with all data
+    if loadAll:
+        print '\nLoading single file with all data...'
+        filename = '%s/%s/%s_allData.json' % (dataFolder, batchLabel, batchLabel)
+        with open(filename, 'r') as fileObj:
+            dataLoad = json.load(fileObj, object_pairs_hook=specs.OrderedDict)
+        params = dataLoad['params']
+        data = dataLoad['data']
+        return params, data
+
+    if isinstance(listCombs, basestring):
+        filename = str(listCombs)
+        with open(filename, 'r') as fileObj:
+            dataLoad = json.load(fileObj)
+        listCombs = dataLoad['paramsMatch']
+
+    # read the batch file and cfg
+    batchFile = '%s/%s_batch.json' % (dataFolder, batchLabel)
+    with open(batchFile, 'r') as fileObj:
+        b = json.load(fileObj)['batch']
+
+    # read params labels and ranges
+    params = b['params']
+
+    # reorder so grouped params come first
+    preorder = [p for p in params if 'group' in p and p['group']]
+    for p in params:
+        if p not in preorder: preorder.append(p)
+    params = preorder
+
+    # read vars from all files - store in dict 
+    if b['method'] == 'grid':
+        labelList, valuesList = zip(*[(p['label'], p['values']) for p in params])
+        valueCombinations = product(*(valuesList))
+        indexCombinations = product(*[range(len(x)) for x in valuesList])
+        data = {}
+        print 'Reading data...'
+        missing = 0
+        for i,(iComb, pComb) in enumerate(zip(indexCombinations, valueCombinations)):
+            if (not maxCombs or i<= maxCombs) and (not listCombs or list(pComb) in listCombs):
+                print i, iComb
+                # read output file
+                iCombStr = ''.join([''.join('_'+str(i)) for i in iComb])
+                simLabel = b['batchLabel']+iCombStr
+                outFile = b['saveFolder']+'/'+simLabel+'.json'
+                try:
+                    with open(outFile, 'r') as fileObj:
+                        output = json.load(fileObj, object_pairs_hook=specs.OrderedDict)
+                    # save output file in data dict
+                    data[iCombStr] = {}  
+                    data[iCombStr]['paramValues'] = pComb  # store param values
+                    if not vars: vars = output.keys()
+
+                    for key in vars:
+                        if isinstance(key, tuple):
+                            container = output
+                            for ikey in range(len(key)-1):
+                                container = container[key[ikey]]
+                            data[iCombStr][key[1]] = container[key[-1]]
+
+                        elif isinstance(key, basestring): 
+                            data[iCombStr][key] = output[key]
+
+                except:
+                    print '... file missing'
+                    missing = missing + 1
+                    output = {}
+            else:
+                missing = missing + 1
+
+        print '%d files missing' % (missing)
+
+        # save
+        if saveAll:
+            print 'Saving to single file with all data'
+            filename = '%s/%s_allData.json' % (dataFolder, batchLabel)
+            dataSave = {'params': params, 'data': data}
+            with open(filename, 'w') as fileObj:
+                json.dump(dataSave, fileObj)
+
+        return params, data
+
+#--------------------------------------------------------------------
+# Function to convert data to Pandas
+#--------------------------------------------------------------------
+def toPandas(params, data):
+    if 'simData' in data[data.keys()[0]]:
+        rows = [list(d['paramValues'])+[s for s in d['simData'].values()] for d in data.values()]
+        cols = [str(d['label']) for d in params]+[s for s in data[data.keys()[0]]['simData'].keys()]
+    else:
+        rows = [list(d['paramValues'])+[s for s in d.values()] for d in data.values()]
+        cols = [str(d['label']) for d in params]+[s for s in data[data.keys()[0]].keys()]
+
+    df = pd.DataFrame(rows, columns=cols) 
+    df['simLabel'] = data.keys()
+
+    colRename=[]
+    for col in list(df.columns):
+        if col.startswith("[u'"):
+            colName = col.replace(", u'","_'").replace("[u","").replace("'","").replace("]","").replace(", ","_")
+            colRename.append(colName)
+        else: 
+            colRename.append(col)
+    print colRename
+    df.columns = colRename
+
+    return df
+
+#--------------------------------------------------------------------
+# Function to colors and style of figures
+#--------------------------------------------------------------------
+def setPlotFormat(numColors=8):
+    plt.style.use('seaborn-whitegrid')
+
+    plt.rcParams['font.size'] = 12
+    plt.rcParams['axes.titlesize'] = 14
+    plt.rcParams['axes.labelsize'] = 12
+    plt.rcParams['legend.fontsize'] = 'large'
+
+    NUM_COLORS = numColors
+    colormap = plt.get_cmap('nipy_spectral')
+    colorlist = [colormap(1.*i/NUM_COLORS) for i in range(NUM_COLORS)]
+
+    plt.rc('axes', prop_cycle=(cycler('color', colorlist)))
+
+
+#--------------------------------------------------------------------
+# Function to plot relation between parameters (tau2 and weight) and firing rate
+#--------------------------------------------------------------------
+def plot2DRate(dataFolder, batchLabel, params, data, par1, par2, val, valLabel, graphType='matrix', saveFile=None):
+    df = toPandas(params, data)
+    # dfpop = dfPopRates(df1, 7)
+
+    dfpop = df.iloc[:,0:5] # get param columns of all rows
+    # dfpop['simLabel'] = df['simLabel']
+    for k in df.popRates[0].keys(): dfpop[k] = [r[k] for r in df.popRates] 
+    #return dfpop
+
+    print dfpop
+    # if not valLabel: valLabel = val
+    dfsubset = dfpop[[par1,par2,val]] 
+        # dfgroup = dfsubset.groupby(by=[par1,par2])
+        # if groupStat=='first':
+        #     dfgroup2 = dfgroup.first()
+        # elif groupStat=='last':
+        #     dfgroup2 = dfgroup.last()
+        # elif groupStat=='mean':
+        #     dfgroup2 = dfgroup.mean()
+        # elif groupStat=='sum':
+        #     dfgroup2 = dfgroup.sum()
+        # dffinal = pd.DataFrame(dfgroup2).reset_index()
+
+    dfpiv = pd.pivot_table(dfsubset, index=par1, columns=par2, values=val)
+#    pandas.pivot_table(df,values='count',index='site_id',columns='week')
+    if graphType=='matrix':
+        sb.heatmap(dfpiv, square=True, cbar_kws={'label': valLabel})
+    elif graphType=='line':
+        setPlotFormat(numColors = len(dfpiv.columns))
+        #dfpiv = dfpiv[['IT2','IT4','IT5A','IT5B','PT5B','IT6','CT6']]
+        dfpiv.plot(marker='o') 
+    try:
+        if saveFile:
+            plt.savefig(saveFile)
+        else:
+            plt.savefig(dataFolder+'/'+batchLabel+'_matrix_'+par1+'_'+par2+'_'+val+'.png')
+    except:
+        print 'Error saving figure...'
+
+    plt.show()
+
+#--------------------------------------------------------------------
+# Function to read batch data and plot figure
+#--------------------------------------------------------------------
+def readPlot():
+    dataFolder = 'tut8_data/'
+    batchLabel = 'tauWeight'
+
+    params, data = readBatchData(dataFolder, batchLabel, loadAll=0, saveAll=1, vars=None, maxCombs=None) 
+    plot2DRate(dataFolder, batchLabel, params, data, 'synMechTau2', 'connWeight', 'M', "'M' pop rate (Hz)")
+
+
+# Main code
+if __name__ == '__main__':
+    readPlot()
+

doc/source/code/tut8_batch.py

@@ -0,0 +1,31 @@
+from netpyne import specs
+from netpyne.batch import Batch 
+
+def batchTauWeight():
+	# Create variable of type ordered dictionary (NetPyNE's customized version) 
+	params = specs.ODict()   
+
+	# fill in with parameters to explore and range of values (key has to coincide with a variable in simConfig) 
+	params['synMechTau2'] = [3.0, 5.0, 7.0]   
+	params['connWeight'] = [0.005, 0.01, 0.15]
+
+	# create Batch object with paramaters to modify, and specifying files to use
+	b = Batch(params=params, cfgFile='tut8_cfg.py', netParamsFile='tut8_netParams.py',)
+
+	# Set output folder, grid method (all param combinations), and run configuration
+	b.batchLabel = 'tauWeight'
+	b.saveFolder = 'tut8_data'
+	b.method = 'grid'
+	b.runCfg = {'type': 'mpi', 
+				'script': 'tut8_init.py', 
+				'skip': True}
+
+	# Run batch simulations
+	b.run()
+
+# Main code
+if __name__ == '__main__':
+	batchTauWeight() 
+
+
+

doc/source/code/tut8_cfg.py

@@ -0,0 +1,19 @@
+from netpyne import specs
+
+# Simulation options
+cfg = specs.SimConfig()		# object of class SimConfig to store simulation configuration
+
+cfg.duration = 1*1e3 			# Duration of the simulation, in ms
+cfg.dt = 0.025 				# Internal integration timestep to use
+cfg.verbose = False  			# Show detailed messages
+cfg.recordTraces = {'V_soma':{'sec':'soma','loc':0.5,'var':'v'}}  # Dict with traces to record
+cfg.recordStep = 0.1 			# Step size in ms to save data (eg. V traces, LFP, etc)
+cfg.filename = 'tut8'  # Set file output name
+cfg.saveJson = True 	
+cfg.printPopAvgRates = True
+cfg.analysis['plotRaster'] = {'saveFig': True} 			# Plot a raster
+cfg.analysis['plotTraces'] = {'include': [20], 'saveFig': True} 			# Plot recorded traces for this list of cells
+
+# Variable parameters (used in netParams)
+cfg.synMechTau2 = 5
+cfg.connWeight = 0.01

doc/source/code/tut8_data/tauWeight_0_0_cfg.json

@@ -0,0 +1,76 @@
+{
+    "simConfig": {
+        "addSynMechs": true, 
+        "analysis": {
+            "plotRaster": {
+                "saveFig": true
+            }, 
+            "plotTraces": {
+                "include": [
+                    20
+                ], 
+                "saveFig": true
+            }
+        }, 
+        "backupCfgFile": [], 
+        "cache_efficient": false, 
+        "checkErrors": false, 
+        "checkErrorsVerbose": false, 
+        "connWeight": 0.005, 
+        "createNEURONObj": true, 
+        "createPyStruct": true, 
+        "cvode_active": false, 
+        "cvode_atol": 0.001, 
+        "dt": 0.025, 
+        "duration": 1000.0, 
+        "filename": "tut8", 
+        "gatherOnlySimData": false, 
+        "hParams": {
+            "celsius": 6.3, 
+            "clamp_resist": 0.001
+        }, 
+        "includeParamsLabel": true, 
+        "printPopAvgRates": true, 
+        "printRunTime": false, 
+        "recordCells": [], 
+        "recordStep": 0.1, 
+        "recordStim": false, 
+        "recordTime": true, 
+        "recordTraces": {
+            "V_soma": {
+                "loc": 0.5, 
+                "sec": "soma", 
+                "var": "v"
+            }
+        }, 
+        "saveCSV": false, 
+        "saveCellConns": true, 
+        "saveCellSecs": true, 
+        "saveDat": false, 
+        "saveDataInclude": [
+            "netParams", 
+            "netCells", 
+            "netPops", 
+            "simConfig", 
+            "simData"
+        ], 
+        "saveDpk": false, 
+        "saveFolder": "tut8_data", 
+        "saveHDF5": false, 
+        "saveJson": true, 
+        "saveMat": false, 
+        "savePickle": false, 
+        "saveTiming": false, 
+        "seeds": {
+            "conn": 1, 
+            "loc": 1, 
+            "stim": 1
+        }, 
+        "simLabel": "tauWeight_0_0", 
+        "synMechTau2": 3.0, 
+        "timestampFilename": false, 
+        "timing": true, 
+        "tstop": 1000.0, 
+        "verbose": false
+    }
+}