Skip to content

Commit

Permalink
Merge branch 'devel' into fix/mpiexec_ppn
Browse files Browse the repository at this point in the history
  • Loading branch information
@mturilli
mturilli authored Oct 30, 2023
2 parents 7a3d8bf + 6170236 commit 9118318
Show file tree
Hide file tree
Showing 7 changed files with 223 additions and 486 deletions.
257 changes: 127 additions & 130 deletions docs/source/internals.rst
View file

Large diffs are not rendered by default.

269 changes: 33 additions & 236 deletions docs/source/tutorials/configuration.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,15 +112,8 @@
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:29.131077Z",
"iopub.status.busy": "2023-05-18T01:28:29.130767Z",
"iopub.status.idle": "2023-05-18T01:28:29.143849Z",
"shell.execute_reply": "2023-05-18T01:28:29.143039Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"resource_tacc_tutorial = \\\n",
Expand Down Expand Up @@ -242,39 +235,17 @@
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:29.147022Z",
"iopub.status.busy": "2023-05-18T01:28:29.146818Z",
"iopub.status.idle": "2023-05-18T01:28:29.151553Z",
"shell.execute_reply": "2023-05-18T01:28:29.150712Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"env: RADICAL_REPORT_ANIME=FALSE\n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"%env RADICAL_REPORT_ANIME=FALSE"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:29.185647Z",
"iopub.status.busy": "2023-05-18T01:28:29.185284Z",
"iopub.status.idle": "2023-05-18T01:28:29.301419Z",
"shell.execute_reply": "2023-05-18T01:28:29.300230Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# ensure that the location for user-defined configurations exists\n",
Expand All @@ -283,15 +254,8 @@
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:29.304970Z",
"iopub.status.busy": "2023-05-18T01:28:29.303986Z",
"iopub.status.idle": "2023-05-18T01:28:29.437958Z",
"shell.execute_reply": "2023-05-18T01:28:29.437066Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"import os\n",
Expand All @@ -310,15 +274,8 @@
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:29.441180Z",
"iopub.status.busy": "2023-05-18T01:28:29.440955Z",
"iopub.status.idle": "2023-05-18T01:28:29.445507Z",
"shell.execute_reply": "2023-05-18T01:28:29.444703Z"
}
},
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"# save earlier defined platform configuration into the user-space\n",
Expand All @@ -327,27 +284,9 @@
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:29.448557Z",
"iopub.status.busy": "2023-05-18T01:28:29.448348Z",
"iopub.status.idle": "2023-05-18T01:28:29.468319Z",
"shell.execute_reply": "2023-05-18T01:28:29.467519Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"schemas : ['local', 'ssh', 'batch', 'interactive']\n",
"resource_manager : SLURM\n",
"cores_per_node : 56\n",
"system_architecture : {'blocked_cores': [], 'blocked_gpus': [], 'options': ['nvme', 'intel'], 'smt': 1}\n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"tutorial_cfg = rp.utils.get_resource_config(resource='tacc.frontera_tutorial')\n",
"\n",
Expand All @@ -357,81 +296,29 @@
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:29.471177Z",
"iopub.status.busy": "2023-05-18T01:28:29.471005Z",
"iopub.status.idle": "2023-05-18T01:28:29.475689Z",
"shell.execute_reply": "2023-05-18T01:28:29.474916Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"job_manager_endpoint : slurm+ssh://frontera.tacc.utexas.edu/\n",
"filesystem_endpoint : sftp://frontera.tacc.utexas.edu/\n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print('job_manager_endpoint : ', rp.utils.get_resource_job_url(resource='tacc.frontera_tutorial', schema='ssh'))\n",
"print('filesystem_endpoint : ', rp.utils.get_resource_fs_url (resource='tacc.frontera_tutorial', schema='ssh'))"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:29.478035Z",
"iopub.status.busy": "2023-05-18T01:28:29.477716Z",
"iopub.status.idle": "2023-05-18T01:28:37.615344Z",
"shell.execute_reply": "2023-05-18T01:28:37.614206Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001B[94mnew session: \u001B[39m\u001B[0m[rp.session.three.mturilli.019495.0003]\u001B[39m\u001B[0m\u001B[94m \\\n",
"database : \u001B[39m\u001B[0m[mongodb://rct-tutorial:****@95.217.193.116:27017/rct-tutorial]\u001B[39m\u001B[0m\u001B[92m ok\n",
"\u001B[39m\u001B[0m\u001B[94mcreate pilot manager\u001B[39m\u001B[0m\u001B[92m ok\n",
"\u001B[39m\u001B[0m"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"session = rp.Session()\n",
"pmgr = rp.PilotManager(session=session)"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:37.618706Z",
"iopub.status.busy": "2023-05-18T01:28:37.618486Z",
"iopub.status.idle": "2023-05-18T01:28:37.625046Z",
"shell.execute_reply": "2023-05-18T01:28:37.624008Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"label : tacc.frontera_tutorial\n",
"launch_methods : {'MPIRUN': {'pre_exec_cached': ['module load TACC']}, 'order': ['MPIRUN']}\n",
"job_manager_endpoint : fork://localhost/\n",
"filesystem_endpoint : file://localhost/\n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"tutorial_cfg = session.get_resource_config(resource='tacc.frontera_tutorial', schema='batch')\n",
"for attr in ['label', 'launch_methods', 'job_manager_endpoint', 'filesystem_endpoint']:\n",
Expand All @@ -448,26 +335,9 @@
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:37.628882Z",
"iopub.status.busy": "2023-05-18T01:28:37.628579Z",
"iopub.status.idle": "2023-05-18T01:28:37.995491Z",
"shell.execute_reply": "2023-05-18T01:28:37.994236Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001B[94msubmit 1 pilot(s)\u001B[39m\u001B[0m\n",
" pilot.0000 tacc.frontera_tutorial 56 cores 0 gpus\u001B[39m\u001B[0m\u001B[92m ok\n",
"\u001B[39m\u001B[0m"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"pd = rp.PilotDescription({\n",
" 'resource' : 'tacc.frontera_tutorial',\n",
Expand All @@ -484,61 +354,9 @@
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:37.998868Z",
"iopub.status.busy": "2023-05-18T01:28:37.998633Z",
"iopub.status.idle": "2023-05-18T01:28:38.003976Z",
"shell.execute_reply": "2023-05-18T01:28:38.003184Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"{'client_sandbox': '/home/mturilli/github/radical.pilot/docs/source/tutorials',\n",
" 'description': {'access_schema': 'batch',\n",
" 'app_comm': [],\n",
" 'candidate_hosts': [],\n",
" 'cleanup': False,\n",
" 'cores': 56,\n",
" 'exit_on_error': False,\n",
" 'gpus': 0,\n",
" 'input_staging': [],\n",
" 'job_name': None,\n",
" 'layout': 'default',\n",
" 'memory': 0,\n",
" 'nodes': 0,\n",
" 'output_staging': [],\n",
" 'prepare_env': {},\n",
" 'project': 'XYZ000',\n",
" 'queue': 'production',\n",
" 'resource': 'tacc.frontera_tutorial',\n",
" 'runtime': 15,\n",
" 'sandbox': None,\n",
" 'services': [],\n",
" 'uid': None},\n",
" 'endpoint_fs': 'file://localhost/',\n",
" 'js_hop': 'fork://localhost/',\n",
" 'js_url': 'fork://localhost/',\n",
" 'log': None,\n",
" 'pilot_sandbox': 'file://localhost/home/mturilli/radical.pilot.sandbox/rp.session.three.mturilli.019495.0003/pilot.0000/',\n",
" 'pmgr': 'pmgr.0000',\n",
" 'resource': 'tacc.frontera_tutorial',\n",
" 'resource_details': None,\n",
" 'resource_sandbox': 'file://localhost/home/mturilli/radical.pilot.sandbox',\n",
" 'session': 'rp.session.three.mturilli.019495.0003',\n",
" 'session_sandbox': 'file://localhost/home/mturilli/radical.pilot.sandbox/rp.session.three.mturilli.019495.0003',\n",
" 'state': 'PMGR_LAUNCHING',\n",
" 'stderr': None,\n",
" 'stdout': None,\n",
" 'type': 'pilot',\n",
" 'uid': 'pilot.0000'}\n"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from pprint import pprint\n",
"\n",
Expand All @@ -555,30 +373,9 @@
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {
"execution": {
"iopub.execute_input": "2023-05-18T01:28:38.006471Z",
"iopub.status.busy": "2023-05-18T01:28:38.006257Z",
"iopub.status.idle": "2023-05-18T01:28:42.732413Z",
"shell.execute_reply": "2023-05-18T01:28:42.731595Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001B[94mclosing session rp.session.three.mturilli.019495.0003\u001B[39m\u001B[0m\u001B[94m \\\n",
"close pilot manager\u001B[39m\u001B[0m\u001B[94m \\\n",
"wait for 1 pilot(s)\n",
" \u001B[39m\u001B[0m\u001B[92m ok\n",
"\u001B[39m\u001B[0m\u001B[92m ok\n",
"\u001B[39m\u001B[0m\u001B[94msession lifetime: 13.1s\u001B[39m\u001B[0m\u001B[92m ok\n",
"\u001B[39m\u001B[0m"
]
}
],
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"session.close(cleanup=True)"
]
Expand All @@ -600,7 +397,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.6"
"version": "3.8.10"
}
},
"nbformat": 4,
Expand Down
4 changes: 1 addition & 3 deletions docs/source/tutorials/profiling.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,9 +175,7 @@
"metadata": {},
"outputs": [],
"source": [
"sessions = ! ls -t | grep rp.session.\n",
"session_id = sessions[0]\n",
"%env SESSION_ID=$session_id"
"%env SESSION_ID=$session.uid"
]
},
{
Expand Down
Loading

0 comments on commit 9118318

Please sign in to comment.