Added script to dump mlperf-run-state, use cache for mlperf results, …

…version info cleanups
mlcommons · Feb 13, 2024 · 2cbf7a7 · 2cbf7a7
1 parent bd5590b
commit 2cbf7a7
Show file tree

Hide file tree

Showing 20 changed files with 124 additions and 195 deletions.
diff --git a/cm-mlops/automation/script/module.py b/cm-mlops/automation/script/module.py
@@ -1316,6 +1316,8 @@ def run(self, i):
                 utils.merge_dicts({'dict1':env, 'dict2':const, 'append_lists':True, 'append_unique':True})
                 utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True})
 
+                run_script_input['run_state'] = run_state
+
                 ii = copy.deepcopy(customize_common_input)
                 ii['env'] = env
                 ii['state'] = state

diff --git a/cm-mlops/script/app-mlperf-inference-cpp/_cm.yaml b/cm-mlops/script/app-mlperf-inference-cpp/_cm.yaml
@@ -151,6 +151,9 @@ post_deps:
       CM_MLPERF_SKIP_RUN:
         - "yes"
 
+  - tags: save,mlperf,inference,state
+    names:
+      - save-mlperf-inference-state
 
 # Variations to customize dependencies
 variations:

diff --git a/cm-mlops/script/app-mlperf-inference-cpp/customize.py b/cm-mlops/script/app-mlperf-inference-cpp/customize.py
@@ -84,37 +84,4 @@ def postprocess(i):
     env = i['env']
     state = i['state']
 
-    if env.get('CM_MLPERF_README', '') == "yes":
-        import cmind as cm
-        inp = i['input']
-        script_tags = inp['tags']
-        script_adr = inp.get('add_deps_recursive', inp.get('adr', {}))
-
-        cm_input = {'action': 'run',
-                'automation': 'script',
-                'tags': script_tags,
-                'adr': script_adr,
-                'print_deps': True,
-                'env': env,
-                'quiet': True,
-                'silent': True,
-                'fake_run': True
-                }
-        r = cm.access(cm_input)
-        if r['return'] > 0:
-            return r
-
-        state['mlperf-inference-implementation'] = {}
-        state['mlperf-inference-implementation']['print_deps'] = r['new_state']['print_deps']
-
-    if env.get('CM_DUMP_VERSION_INFO', True):
-        if not state.get('mlperf-inference-implementation', {}):
-            state['mlperf-inference-implementation'] = {}
-        run_state = i['run_script_input']['run_state']
-        state['mlperf-inference-implementation'][run_state['script_uid']] = {}
-        version_info = {}
-        version_info[run_state['script_uid']] = run_state['version_info']
-
-        state['mlperf-inference-implementation']['version_info'] = version_info
-
     return {'return':0}
diff --git a/cm-mlops/script/app-mlperf-inference-reference/_cm.yaml b/cm-mlops/script/app-mlperf-inference-reference/_cm.yaml
@@ -574,7 +574,10 @@ posthook_deps:
       CM_MLPERF_SKIP_RUN:
       - "on"
 
-
+post_deps:
+  - tags: save,mlperf,inference,state
+    names:
+      - save-mlperf-inference-state
 
 # Variations to customize dependencies
 variations:

diff --git a/cm-mlops/script/app-mlperf-inference-reference/customize.py b/cm-mlops/script/app-mlperf-inference-reference/customize.py
@@ -363,36 +363,6 @@ def postprocess(i):
     env = i['env']
     state = i['state']
 
-    if env.get('CM_MLPERF_README', "") == "yes":
-        import cmind as cm
-        inp = i['input']
-        script_tags = inp['tags']
-        script_adr = inp.get('add_deps_recursive', inp.get('adr', {}))
-
-        cm_input = {'action': 'run',
-                'automation': 'script',
-                'tags': script_tags,
-                'adr': script_adr,
-                'env': env,
-                'print_deps': True,
-                'quiet': True,
-                'silent': True,
-                'fake_run': True
-                }
-        r = cm.access(cm_input)
-        if r['return'] > 0:
-            return r
-
-        state['mlperf-inference-implementation'] = {}
-        state['mlperf-inference-implementation']['print_deps'] = r['new_state']['print_deps']
-
-    if env.get('CM_DUMP_VERSION_INFO', True):
-        if not state.get('mlperf-inference-implementation', {}):
-            state['mlperf-inference-implementation'] = {}
-        run_state = i['run_script_input']['run_state']
-        version_info = {}
-        version_info[run_state['script_id']+":"+",".join(run_state['script_variation_tags'])] = run_state['version_info']
-
-        state['mlperf-inference-implementation']['version_info'] = version_info
+    inp = i['input']
 
     return {'return':0}
diff --git a/cm-mlops/script/app-mlperf-inference-tflite-cpp/_cm.json b/cm-mlops/script/app-mlperf-inference-tflite-cpp/_cm.json
@@ -252,6 +252,12 @@
           "yes"
         ]
       }
+    },
+    {
+      "tags": "save,mlperf,inference,state",
+      "names": [
+        "save-mlperf-inference-state"
+      ]
     }
   ],
   "tags": [

diff --git a/cm-mlops/script/app-mlperf-inference-tflite-cpp/customize.py b/cm-mlops/script/app-mlperf-inference-tflite-cpp/customize.py
@@ -93,37 +93,4 @@ def postprocess(i):
     env = i['env']
     state = i['state']
 
-    if env.get('CM_MLPERF_README', '') == "yes":
-        import cmind as cm
-        inp = i['input']
-        script_tags = inp['tags']
-        script_adr = inp.get('add_deps_recursive', inp.get('adr', {}))
-
-        cm_input = {'action': 'run',
-                'automation': 'script',
-                'tags': script_tags,
-                'adr': script_adr,
-                'env': env,
-                'print_deps': True,
-                'quiet': True,
-                'silent': True,
-                'fake_run': True
-                }
-        r = cm.access(cm_input)
-        if r['return'] > 0:
-            return r
-
-        state['mlperf-inference-implementation'] = {}
-        state['mlperf-inference-implementation']['print_deps'] = r['new_state']['print_deps']
-
-    if env.get('CM_DUMP_VERSION_INFO', True):
-        if not state.get('mlperf-inference-implementation', {}):
-            state['mlperf-inference-implementation'] = {}
-        run_state = i['run_script_input']['run_state']
-        state['mlperf-inference-implementation'][run_state['script_uid']] = {}
-        version_info = {}
-        version_info[run_state['script_uid']] = run_state['version_info']
-
-        state['mlperf-inference-implementation']['version_info'] = version_info
-
     return {'return':0}
diff --git a/cm-mlops/script/app-mlperf-inference/customize.py b/cm-mlops/script/app-mlperf-inference/customize.py
@@ -12,6 +12,7 @@
 def preprocess(i):
 
     env = i['env']
+    state = i['state']
 
     if env.get('CM_MLPERF_IMPLEMENTATION', '') == 'nvidia-original':
         if env.get('CM_NVIDIA_GPU_NAME', '') in [ "rtx_4090", "a100", "t4", "l4", "orin", "custom" ]:
@@ -24,7 +25,12 @@ def preprocess(i):
             env['CM_NVIDIA_HARNESS_GPU_VARIATION'] = ''
 
     if 'cmd' in i['input']:
-        i['state']['mlperf_inference_run_cmd'] = "cm run script " + " ".join(i['input']['cmd'])
+        state['mlperf_inference_run_cmd'] = "cm run script " + " ".join(i['input']['cmd'])
+
+    state['mlperf-inference-implementation'] = {}
+
+    run_state = i['run_script_input']['run_state']
+    state['mlperf-inference-implementation']['script_id'] = run_state['script_id']+":"+",".join(run_state['script_variation_tags'])
 
     return {'return':0}
 

diff --git a/cm-mlops/script/generate-mlperf-inference-user-conf/_cm.yaml b/cm-mlops/script/generate-mlperf-inference-user-conf/_cm.yaml
@@ -73,7 +73,11 @@ deps:
     - python
     - python3
 
-
+  - tags: get,mlperf,results,dir
+    names:
+      - get-mlperf-results-dir
+    skip_if_env:
+      OUTPUT_BASE_DIR: [ on ]
 
   ########################################################################
   # Install MLPerf inference dependencies

diff --git a/cm-mlops/script/generate-mlperf-inference-user-conf/customize.py b/cm-mlops/script/generate-mlperf-inference-user-conf/customize.py
@@ -35,8 +35,8 @@ def preprocess(i):
         env['CM_MLPERF_LOADGEN_MODE'] = "accuracy"
 
 
-    if 'OUTPUT_BASE_DIR' not in env:
-        env['OUTPUT_BASE_DIR'] = os.getcwd()
+    if env.get('OUTPUT_BASE_DIR', '') == '':
+        env['OUTPUT_BASE_DIR'] = env.get('CM_MLPERF_INFERENCE_RESULTS_DIR', os.getcwd())
 
     if 'CM_NUM_THREADS' not in env:
         if 'CM_MINIMIZE_THREADS' in env:

diff --git a/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml
@@ -160,6 +160,9 @@ post_deps:
         - run_harness
     tags: benchmark-mlperf
 
+  - tags: save,mlperf,inference,state
+    names:
+      - save-mlperf-inference-state
 
 # Variations to customize dependencies
 variations:

diff --git a/cm-mlops/script/reproduce-mlperf-inference-intel/customize.py b/cm-mlops/script/reproduce-mlperf-inference-intel/customize.py
@@ -94,37 +94,4 @@ def postprocess(i):
     env = i['env']
     state = i['state']
 
-    if env.get('CM_MLPERF_README', '') == "yes":
-        import cmind as cm
-        inp = i['input']
-        script_tags = inp['tags']
-        script_adr = inp.get('add_deps_recursive', inp.get('adr', {}))
-
-        cm_input = {'action': 'run',
-                'automation': 'script',
-                'tags': script_tags,
-                'adr': script_adr,
-                'print_deps': True,
-                'env': env,
-                'quiet': True,
-                'silent': True,
-                'fake_run': True
-                }
-        r = cm.access(cm_input)
-        if r['return'] > 0:
-            return r
-
-        state['mlperf-inference-implementation'] = {}
-        state['mlperf-inference-implementation']['print_deps'] = r['new_state']['print_deps']
-
-    if env.get('CM_DUMP_VERSION_INFO', True):
-        if not state.get('mlperf-inference-implementation', {}):
-            state['mlperf-inference-implementation'] = {}
-        run_state = i['run_script_input']['run_state']
-        state['mlperf-inference-implementation'][run_state['script_uid']] = {}
-        version_info = {}
-        version_info[run_state['script_uid']] = run_state['version_info']
-
-        state['mlperf-inference-implementation']['version_info'] = version_info
-
     return {'return':0}
diff --git a/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-nvidia/_cm.yaml
@@ -291,7 +291,9 @@ post_deps:
     enable_if_env:
       CM_CALL_MLPERF_RUNNER:
         - yes
-
+  - tags: save,mlperf,inference,state
+    names:
+      - save-mlperf-inference-state
 
 # Variations to customize dependencies
 variations:

diff --git a/cm-mlops/script/reproduce-mlperf-inference-nvidia/customize.py b/cm-mlops/script/reproduce-mlperf-inference-nvidia/customize.py
@@ -418,37 +418,4 @@ def postprocess(i):
     env = i['env']
     state = i['state']
 
-    if env.get('CM_MLPERF_README', '') == "yes":
-        import cmind as cm
-        inp = i['input']
-        script_tags = inp['tags']
-        script_adr = inp.get('add_deps_recursive', inp.get('adr', {}))
-
-        cm_input = {'action': 'run',
-                'automation': 'script',
-                'tags': script_tags,
-                'adr': script_adr,
-                'env': env,
-                'print_deps': True,
-                'quiet': True,
-                'silent': True,
-                'fake_run': True
-                }
-        r = cm.access(cm_input)
-        if r['return'] > 0:
-            return r
-
-        state['mlperf-inference-implementation'] = {}
-        state['mlperf-inference-implementation']['print_deps'] = r['new_state']['print_deps']
-
-    if env.get('CM_DUMP_VERSION_INFO', True):
-        if not state.get('mlperf-inference-implementation', {}):
-            state['mlperf-inference-implementation'] = {}
-        run_state = i['run_script_input']['run_state']
-        state['mlperf-inference-implementation'][run_state['script_uid']] = {}
-        version_info = {}
-        version_info[run_state['script_uid']] = run_state['version_info']
-
-        state['mlperf-inference-implementation']['version_info'] = version_info
-
     return {'return':0}
diff --git a/cm-mlops/script/reproduce-mlperf-inference-qualcomm/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-qualcomm/_cm.yaml
@@ -261,6 +261,9 @@ post_deps:
         - yes
     tags: benchmark-mlperf
 
+  - tags: save,mlperf,inference,state
+    names:
+      - save-mlperf-inference-state
 
 # Variations to customize dependencies
 variations:

diff --git a/cm-mlops/script/reproduce-mlperf-inference-qualcomm/customize.py b/cm-mlops/script/reproduce-mlperf-inference-qualcomm/customize.py
@@ -183,28 +183,5 @@ def preprocess(i):
 def postprocess(i):
 
     env = i['env']
-    if env.get('CM_MLPERF_README', '') == "yes":
-        import cmind as cm
-        inp = i['input']
-        state = i['state']
-        script_tags = inp['tags']
-        script_adr = inp.get('add_deps_recursive', inp.get('adr', {}))
-
-        cm_input = {'action': 'run',
-                'automation': 'script',
-                'tags': script_tags,
-                'adr': script_adr,
-                'print_deps': True,
-                'env': env,
-                'quiet': True,
-                'silent': True,
-                'fake_run': True
-                }
-        r = cm.access(cm_input)
-        if r['return'] > 0:
-            return r
-
-        state['mlperf-inference-implementation'] = {}
-        state['mlperf-inference-implementation']['print_deps'] = r['new_state']['print_deps']
 
     return {'return':0}
diff --git a/cm-mlops/script/run-mlperf-inference-app/_cm.yaml b/cm-mlops/script/run-mlperf-inference-app/_cm.yaml
@@ -103,6 +103,12 @@ deps:
   tags: get,mlcommons,inference,src
 - tags: get,sut,description
 
+- tags: get,mlperf,inference,results,dir
+  names:
+    - get-mlperf-inference-results-dir
+  skip_if_env:
+    OUTPUT_BASE_DIR: [ on ]
+
 docker:
   fake_run_deps: true
   mounts:

diff --git a/cm-mlops/script/run-mlperf-inference-app/customize.py b/cm-mlops/script/run-mlperf-inference-app/customize.py
@@ -92,9 +92,8 @@ def preprocess(i):
     else:
         env['CM_MLPERF_LOADGEN_MODES'] = [ env['CM_MLPERF_LOADGEN_MODE'] ]
 
-
-    if 'OUTPUT_BASE_DIR' not in env:
-        env['OUTPUT_BASE_DIR'] = os.getcwd()
+    if env.get('OUTPUT_BASE_DIR', '') == '':
+        env['OUTPUT_BASE_DIR'] = env.get('CM_MLPERF_INFERENCE_RESULTS_DIR', os.getcwd())
 
     test_list = ["TEST01",  "TEST05"]
     if env['CM_MODEL']  in ["resnet50"]:
@@ -139,6 +138,7 @@ def preprocess(i):
         print ('=========================================================')
 
     local_keys = [ 'CM_MLPERF_SKIP_RUN', 'CM_MLPERF_LOADGEN_QUERY_COUNT' ]
+
     for scenario in env['CM_MLPERF_LOADGEN_SCENARIOS']:
         scenario_tags = tags + ",_"+scenario.lower()
         env['CM_MLPERF_LOADGEN_SCENARIO'] = scenario

diff --git a/cm-mlops/script/save-mlperf-inference-implementation-state/_cm.yaml b/cm-mlops/script/save-mlperf-inference-implementation-state/_cm.yaml
@@ -0,0 +1,13 @@
+alias: save-mlperf-inference-implementation-state
+automation_alias: script
+automation_uid: 5b4e0237da074764
+cache: false
+new_state_keys:
+  - mlperf-inference-implementation
+tags:
+- save
+- mlperf
+- inference
+- implementation
+- state
+uid: b14b813229c444f8