merge develop and update gradle

LBNL-UCB-STI · Aug 24, 2022 · e89d6a8 · e89d6a8
2 parents aa1f7b4 + 07a9f07
commit e89d6a8
Show file tree

Hide file tree

Showing 435 changed files with 8,629 additions and 3,814 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -35,3 +35,4 @@ text eol=lf
 test/**/*.xml filter=lfs diff=lfs merge=lfs -text
 production/**/*.xml filter=lfs diff=lfs merge=lfs -text
 test/input/sf-light/TechTakeover_longTerm_lowTech/ filter=lfs diff=lfs merge=lfs -text
+aws/**/*.txt !filter !diff !merge text
diff --git a/.gitignore b/.gitignore
@@ -56,3 +56,4 @@ src/main/scala/beam/sandbox
 test/scala/beam/sandbox
 log-path_IS_UNDEFINED/
 .Rproj.user
+thread_dump_from_RunBeam.txt.gz
diff --git a/aws/build.gradle b/aws/build.gradle
@@ -1,8 +1,6 @@
 import com.amazonaws.services.lambda.model.InvocationType
-import com.amazonaws.services.lambda.model.InvokeResult
 import jp.classmethod.aws.reboot.gradle.lambda.AWSLambdaInvokeTask
 
-
 buildscript {
     repositories {
         mavenLocal()
@@ -80,14 +78,16 @@ task deploy(type: AWSLambdaInvokeTask) {
         def tempInstanceType = "${ext.getParameterValue('instanceType') ?: (project.hasProperty('defaultInstanceType') ? defaultInstanceType : '')}"
         def finalInstanceType = tempInstanceType != "" ? tempInstanceType : null
         def dataBranch = "${ext.getParameterValue('dataBranch') ?: 'develop'}"
+        def dataCommit = "${ext.getParameterValue('dataCommit') ?: 'HEAD'}"
 
         def pload = """{
   "title": "${ext.getParameterValue('runName') + '_' + getCurrentGitUserEmail()}",
   "git_user_email": "${getCurrentGitUserEmail()}",
   "deploy_type_tag": "${ext.getParameterValue('deployTypeTag')}",
   "branch": "${ext.getParameterValue('beamBranch') ?: getCurrentGitBranch()}",
-  "data_branch": "$dataBranch",
   "commit": "${ext.getParameterValue('beamCommit') ?: 'HEAD'}",
+  "data_branch": "$dataBranch",
+  "data_commit": "$dataCommit",
   "deploy_mode": "${ext.getParameterValue('deployMode')}",
   "configs": "${ext.getParameterValue('beamConfigs') ?: ext.getParameterValue(getCurrentGitBranch() + '.configs')}",
   "experiments": "${ext.getParameterValue('beamExperiments') ?: ext.getParameterValue(getCurrentGitBranch() + '.experiments')}",
@@ -105,14 +105,17 @@ task deploy(type: AWSLambdaInvokeTask) {
   "command": "deploy",
   "run_grafana" : ${ext.getParameterValue('runGrafana') ?: false},
   "run_helics" : ${ext.getParameterValue('runHelics') ?: false},
+  "run_jupyter" : ${ext.getParameterValue('runJupyter') ?: false},
+  "jupyter_token" : "${ext.getParameterValue('jupyterToken') ?: UUID.randomUUID().toString()}",
   "is_spot" : ${ext.getParameterValue('isSpot') ?: false},
   "min_cores": ${ext.getParameterValue('minCores') ?: 0},
   "max_cores": ${ext.getParameterValue('maxCores') ?: 0},
   "min_memory": ${ext.getParameterValue('minMemory') ?: 0},
-  "max_memory": ${ext.getParameterValue('maxMemory') ?: 0}
+  "max_memory": ${ext.getParameterValue('maxMemory') ?: 0},
+  "budget_override": ${ext.getParameterValue('budgetOverride') ?: false}
 }"""
         payload = pload
-        println "Using data branch $dataBranch"
+        println "Using data branch $dataBranch at commit $dataCommit"
         println payload
         println "Please note that if you set isSpot to true then it could take an excessive time period. In fact it could time out at 15 minutes and still complete later on. ALSO! volumes using spot instances must be MANUALLY deleted. This is done so you do not lose data in the case of a premature shutdown."
         functionName = "simulateBeam"
@@ -147,15 +150,18 @@ task deployPilates(type: AWSLambdaInvokeTask) {
         defaultParamVals['region'] = defRegion
         defaultParamVals['dataRegion'] = defRegion
         defaultParamVals['beamBranch'] = getCurrentGitBranch()
+        defaultParamVals['beamCommit'] = 'HEAD'
         defaultParamVals['dataBranch'] = 'develop'
+        defaultParamVals['dataCommit'] = 'HEAD'
+        defaultParamVals['budgetOverride'] = False
 
         def paramMap = [:]
         def missing = []
 
         // all possible parameters
         ['runName', 'pilatesScenarioName',
-         'beamBranch', 'dataBranch', 'beamCommit',
-         'beamConfig',
+         'beamBranch', 'beamCommit', 'dataBranch', 'dataCommit',
+         'beamConfig', 'budgetOverride',
          'startYear', 'countOfYears', 'beamItLen', 'urbansimItLen',
          'initialS3UrbansimInput', 'initialS3UrbansimOutput', 'initialSkimPath',
          'pilatesImageVersion', 'pilatesImageName',
@@ -174,10 +180,11 @@ task deployPilates(type: AWSLambdaInvokeTask) {
         }
 
         paramMap['runName'] = paramMap['runName'] + '_' + getCurrentGitUserEmail()
+        paramMap['budget_override'] = paramMap['budgetOverride']
 
         println("parameters wasn't specified: " + missing.join(', '))
 
-        println "Using data branch ${paramMap['dataBranch']}"
+        println "Using data branch ${paramMap['dataBranch']} at commit ${paramMap['dataCommit']}"
 
         // create a string with json object which contains all parameters
         def pload = "{\n" + paramMap.collect { key, val -> "   \"$key\": \"$val\"" }.join(",\n") + "\n}"
@@ -235,6 +242,35 @@ task terminateEC2(type: AWSLambdaInvokeTask) {
     }
 }
 
+task jupyterEC2(type: AWSLambdaInvokeTask) {
+    doFirst {
+        functionName = "simulateBeam"
+        invocationType = InvocationType.RequestResponse
+        payload = """{
+          "title": "${(project.findProperty('title') ?: 'jupyter') + '_' + getCurrentGitUserEmail()}",
+          "git_user_email": "${getCurrentGitUserEmail()}",
+          "branch": "${project.findProperty('beamBranch') ?: getCurrentGitBranch()}",
+          "storage_size": ${project.findProperty('storageSize') ?: 256},
+          "instance_type": "${project.findProperty('instanceType') ?: 'r5.xlarge'}",
+          "region": "${project.findProperty('region') ?: defaultRegion}",
+          "shutdown_behaviour": "${project.findProperty('shutdownBehaviour') ?: "terminate"}",
+          "shutdown_wait": ${project.findProperty('shutdownWait') ?: 480},
+          "command": "deploy",
+          "run_grafana" : false,
+          "run_helics" : false,
+          "run_jupyter" : true,
+          "run_beam": false,
+          "jupyter_token" : "${project.findProperty("jupyterToken") ?: UUID.randomUUID().toString()}",
+          "is_spot" : false,
+          "budget_override": ${project.findProperty('budgetOverride') ?: false}
+        }"""
+
+    }
+    doLast {
+        printResult(payload, invokeResult)
+    }
+}
+
 task listS3Bucket(type: AWSLambdaInvokeTask) {
     doFirst {
         functionName = "s3BucketList"
@@ -267,6 +303,7 @@ def buildPayload(command) {
     return """{
   "instance_ids": "${project.findProperty('instanceIds')}",
   "region": "${project.findProperty('region') ?: defaultRegion}",
+  "budget_override": ${project.findProperty('budgetOverride') ?: false},
   "command": "$command"
 }"""
 }