Merge from CTuning (More info added for MLPerf results)

cm-mlops/automation/script/module.py

@@ -30,7 +30,7 @@ def __init__(self, cmind, automation_file):
         self.run_state['deps'] = []
         self.run_state['fake_deps'] = False
         self.run_state['parent'] = None
-        self.run_state['version_info'] = {}
+        self.run_state['version_info'] = []
 
         self.file_with_cached_state = 'cm-cached-state.json'
 
@@ -289,8 +289,8 @@ def run(self, i):
         if fake_deps: env['CM_TMP_FAKE_DEPS']='yes'
 
         run_state = i.get('run_state', self.run_state)
-        if run_state.get('version_info', '') == '':
-            run_state['version_info'] = {}
+        if not run_state.get('version_info', []):
+            run_state['version_info'] = []
         if run_state.get('parent', '') == '':
             run_state['parent'] = None
         if fake_deps:
@@ -643,7 +643,9 @@ def run(self, i):
         if i.get('help',False):
             return utils.call_internal_module(self, __file__, 'module_help', 'print_help', {'meta':meta, 'path':path})
 
-
+        run_state['script_id'] = meta['alias'] + "," + meta['uid']
+        run_state['script_variation_tags'] = variation_tags
+
         deps = meta.get('deps',[])
         post_deps = meta.get('post_deps',[])
         prehook_deps = meta.get('prehook_deps',[])
@@ -1314,6 +1316,8 @@ def run(self, i):
                 utils.merge_dicts({'dict1':env, 'dict2':const, 'append_lists':True, 'append_unique':True})
                 utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True})
 
+                run_script_input['run_state'] = run_state
+
                 ii = copy.deepcopy(customize_common_input)
                 ii['env'] = env
                 ii['state'] = state
@@ -1582,22 +1586,26 @@ def run(self, i):
 
         if not version and detected_version:
           version = detected_version
+
         if version:
             script_uid = script_artifact.meta.get('uid')
             script_alias = script_artifact.meta.get('alias')
             script_tags = script_artifact.meta.get('tags')
-            tags = i.get('tags')
-            run_state['version_info'][script_uid] = {}
-            run_state['version_info'][script_uid]['alias'] = script_alias
-            run_state['version_info'][script_uid]['script_tags'] = script_tags
-            run_state['version_info'][script_uid]['variation_tags'] = variation_tags
-            run_state['version_info'][script_uid]['version'] = version
-
+            version_info = {}
+            version_info_tags = ",".join(script_tags + variation_tags)
+            version_info[version_info_tags] = {}
+            version_info[version_info_tags]['script_uid'] = script_uid
+            version_info[version_info_tags]['script_alias'] = script_alias
+            version_info[version_info_tags]['version'] = version
+            version_info[version_info_tags]['parent'] = run_state['parent']
+            run_state['version_info'].append(version_info)
             script_versions = detected_versions.get(meta['uid'], [])
             if not script_versions:
                 detected_versions[meta['uid']] = [ version ]
             else:
                 script_versions.append(version)
+        else:
+            pass # these scripts don't have versions. Should we use cm mlops version here?
 
         ############################# RETURN
         elapsed_time = time.time() - start_time
@@ -1617,6 +1625,11 @@ def run(self, i):
             with open('readme.md', 'w') as f:
                 f.write(readme)
 
+        if i.get('dump_version_info'):
+            r = self._dump_version_info_for_script()
+            if r['return'] > 0:
+                return r
+
         rr = {'return':0, 'env':env, 'new_env':new_env, 'state':state, 'new_state':new_state, 'deps': run_state['deps']}
 
         if i.get('json', False) or i.get('j', False):
@@ -1631,6 +1644,12 @@ def run(self, i):
 
         return rr
 
+    def _dump_version_info_for_script(self, output_dir = os.getcwd()):
+        import json
+        with open(os.path.join(output_dir, 'version_info.json'), 'w') as f:
+            f.write(json.dumps(self.run_state['version_info'], indent=2))
+        return {'return': 0}
+
     def _update_state_from_variations(self, i, meta, variation_tags, variations, env, state, deps, post_deps, prehook_deps, posthook_deps, new_env_keys_from_meta, new_state_keys_from_meta, add_deps_recursive, run_state, recursion_spaces, verbose):
 
         # Save current explicit variations
@@ -2686,7 +2705,9 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
                     tmp_run_state_deps = copy.deepcopy(run_state['deps'])
                     run_state['deps'] = []
                     tmp_parent = run_state['parent']
-                    run_state['parent'] = self.meta['uid']
+                    run_state['parent'] = run_state['script_id']+":"+",".join(run_state['script_variation_tags'])
+                    tmp_script_id = run_state['script_id']
+                    tmp_script_variation_tags = run_state['script_variation_tags']
 
                     # Run collective script via CM API:
                     # Not very efficient but allows logging - can be optimized later
@@ -2722,12 +2743,13 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
 
                     run_state['deps'] = tmp_run_state_deps
                     run_state['parent'] = tmp_parent
+                    run_state['script_id'] = tmp_script_id
+                    run_state['script_variation_tags'] = tmp_script_variation_tags
 
                     # Restore local env
                     env.update(tmp_env)
                     update_env_with_values(env)
 
-
         return {'return': 0}
 
     ##############################################################################
@@ -3974,6 +3996,8 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
     verbose = i.get('verbose', False)
     if not verbose: verbose = i.get('v', False)
 
+    show_time = i.get('time', False)
+
     recursion = i.get('recursion', False)
     found_script_tags = i.get('found_script_tags', [])
     debug_script_tags = i.get('debug_script_tags', '')
@@ -4143,10 +4167,9 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
     if customize_code is not None:
         print (recursion_spaces+'       ! call "{}" from {}'.format(postprocess, customize_code.__file__))
 
-
     if len(posthook_deps)>0 and (postprocess == "postprocess"):
         r = script_automation._call_run_deps(posthook_deps, local_env_keys, local_env_keys_from_meta, env, state, const, const_state,
-            add_deps_recursive, recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, run_state)
+            add_deps_recursive, recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, show_time, ' ', run_state)
         if r['return']>0: return r
 
     if (postprocess == "postprocess") and customize_code is not None and 'postprocess' in dir(customize_code):

cm-mlops/automation/utils/module.py

@@ -878,3 +878,81 @@ def uid(self, i):
 
         return r
 
+
+    ##############################################################################
+    def system(self, i):
+        """
+        Run system command and redirect output to string.
+
+        Args:
+          (CM input dict): 
+
+          * cmd (str): command line
+          * (path) (str): go to this directory and return back to current
+          * (stdout) (str): stdout file
+          * (stderr) (str): stderr file
+
+        Returns:
+          (CM return dict):
+
+          * return (int): return code == 0 if no error and >0 if error
+          * (error) (str): error string if return>0
+
+          * ret (int): return code
+          * std (str): stdout + stderr
+          * stdout (str): stdout
+          * stderr (str): stderr
+        """
+
+        cmd = i['cmd']
+
+        if cmd == '':
+            return {'return':1, 'error': 'cmd is empty'}
+
+        path = i.get('path','')
+        if path!='' and os.path.isdir(path):
+            cur_dir = os.getcwd()
+            os.chdir(path)    
+
+        if i.get('stdout','')!='':
+            fn1=i['stdout']
+            fn1_delete = False
+        else:
+            r = utils.gen_tmp_file({})
+            if r['return'] > 0: return r
+            fn1 = r['file_name']
+            fn1_delete = True
+
+        if i.get('stderr','')!='':
+            fn2=i['stderr']
+            fn2_delete = False
+        else:
+            r = utils.gen_tmp_file({})
+            if r['return'] > 0: return r
+            fn2 = r['file_name']
+            fn2_delete = True
+
+        cmd += ' > '+fn1 + ' 2> '+fn2
+        rx = os.system(cmd)
+
+        std = ''
+        stdout = ''
+        stderr = ''
+
+        if os.path.isfile(fn1):
+            r = utils.load_txt(file_name = fn1, remove_after_read = fn1_delete)
+            if r['return'] == 0: stdout = r['string'].strip()
+
+        if os.path.isfile(fn2):
+            r = utils.load_txt(file_name = fn2, remove_after_read = fn2_delete)
+            if r['return'] == 0: stderr = r['string'].strip()
+
+        std = stdout
+        if stderr!='':
+            if std!='': std+='\n'
+            std+=stderr
+
+        if path!='' and os.path.isdir(path):
+            os.chdir(cur_dir)
+
+        return {'return':0, 'ret':rx, 'stdout':stdout, 'stderr':stderr, 'std':std}

cm-mlops/cfg/benchmark-hardware-compute/amd-gpu.json

@@ -1,4 +1,5 @@
 {
   "uid": "d8f06040f7294319",
-  "name": "AMD GPU"
+  "name": "AMD GPU",
+  "tags": "gpu,amd"
 }

cm-mlops/cfg/benchmark-hardware-compute/generic-cpu-arm64.json

@@ -1,4 +1,5 @@
 {
   "uid":"357a972e79614903",
-  "name": "Generic CPU - Arm64"
+  "name": "Generic CPU - Arm64",
+  "tags": "cpu,arm64,generic"
 }

cm-mlops/cfg/benchmark-hardware-compute/generic-cpu-x64.json

@@ -1,4 +1,5 @@
 {
   "uid": "cdfd424c32734e38",
-  "name": "Generic CPU - x64"
+  "name": "Generic CPU - x64",
+  "tags": "cpu,x64,generic"
 }

cm-mlops/cfg/benchmark-hardware-compute/google-tpu.json

@@ -1,4 +1,5 @@
 {
   "uid": "b3be7ac9ef954f5a",
-  "name": "Google TPU"
+  "name": "Google TPU",
+  "tags": "tpu,google"
 }

cm-mlops/cfg/benchmark-hardware-compute/nvidia-gpu-jetson-orin.json

@@ -1,4 +1,5 @@
 {
   "uid": "fe379ecd1e054a00",
-  "name": "Nvidia GPU - Jetson Orin"
+  "name": "Nvidia GPU - Jetson Orin",
+  "tags": "gpu,nvidia,jetson,orin"
 }

cm-mlops/cfg/benchmark-hardware-compute/nvidia-gpu.json

@@ -1,4 +1,5 @@
 {
   "uid": "fe379ecd1e054a00",
-  "name": "Nvidia GPU"
+  "name": "Nvidia GPU",
+  "tags": "gpu,nvidia"
 }

cm-mlops/cfg/benchmark-hardware-compute/qualcomm-ai100.json

@@ -1,4 +1,5 @@
 {
-  "uid": "fe379ecd1e054a00",
-  "name": "Qualcomm - AI 100"
+  "uid": "d2ae645066664463",
+  "name": "Qualcomm - AI 100",
+  "tags": "accelerator,acc,qualcomm,ai,100,ai-100"
 }

cm-mlops/cfg/benchmark-run-loadgen-cpp-dev/any-model-x64-offline.yaml

@@ -0,0 +1,5 @@
+uid: 125abafe58dc4473
+
+name: "Any model - x64 - offline"
+
+compute_uid: cdfd424c32734e38

cm-mlops/cfg/benchmark-run-loadgen-python-dev/any-model-offline.yaml

@@ -2,6 +2,4 @@ uid: db45dcd686854602
 
 name: "Any model - offline"
 
-supported_compute:
-- cdfd424c32734e38
-- 357a972e79614903
+compute_uid: cdfd424c32734e38

cm-mlops/cfg/benchmark-run-mlperf-abtf-dev/retinanet-reference-python-torch-offline.yaml

@@ -2,5 +2,4 @@ uid: "fe379ecd1e054a00"
 
 name: "RetinaNet Reference Python Torch Offline"
 
-supported_compute:
-- cdfd424c32734e38
+compute_uid: cdfd424c32734e38

cm-mlops/cfg/benchmark-run-mlperf-inference-latest/_cm.yaml

@@ -16,6 +16,7 @@ name: "MLPerf inference - latest"
 supported_compute:
 - 357a972e79614903
 - cdfd424c32734e38
+- d2ae645066664463
 
 urls:
 - name: "Official page"
@@ -24,3 +25,11 @@ urls:
   url: "https://github.com/mlcommons/inference"
 - name: "MLCommons CM automation (under development)"
   url: "https://github.com/mlcommons/ck/tree/master/docs/mlperf/inference"
+
+dimensions:
+- - input.model
+  - "MLPerf model"
+- - input.implementation
+  - "MLPerf implementation"
+- - input.framework
+  - "MLPerf framework"

cm-mlops/cfg/benchmark-run-mlperf-inference-scc23/_cm.yaml

@@ -0,0 +1,26 @@
+alias: benchmark-run-mlperf-inference-scc23
+uid: 9133e5b1dddc4e4a
+
+automation_alias: cfg
+automation_uid: 88dce9c160324c5d
+
+tags:
+- benchmark
+- run
+- mlperf
+- inference
+- v3.1
+
+name: "MLPerf inference - Student Cluster Competition 2023"
+
+supported_compute:
+- fe379ecd1e054a00
+- cdfd424c32734e38
+- fe379ecd1e054a00
+- d2ae645066664463
+
+urls:
+- name: "Official page"
+  url: "https://sc23.supercomputing.org/students/student-cluster-competition/"
+- name: "Tutorial to run MLPerf inference benchmark "
+  url: "https://github.com/mlcommons/ck/blob/master/docs/tutorials/scc23-mlperf-inference-bert.md"

cm-mlops/cfg/benchmark-run-mlperf-inference-scc23/base/_test.yaml

@@ -0,0 +1,3 @@
+name: "BASE"
+
+tags: "base"

cm-mlops/cfg/benchmark-run-mlperf-inference-scc23/bert-reference-python-onnx-offline.md

@@ -0,0 +1 @@
+TBD