diff --git a/README.md b/README.md
index 7cbf3b65e2..d447da2cfd 100755
--- a/README.md
+++ b/README.md
@@ -25,7 +25,7 @@ either natively or inside containers.
Here are a few most commonly used examples from the CM users
that should run in the same way on Linux, MacOS, Windows and other platforms
-(see [Gettings Started Guide](docs/getting-started.md) to understand
+(see [Getting Started Guide](docs/getting-started.md) to understand
how they work and how to reuse them in your projects):
```bash
@@ -38,6 +38,7 @@ cmr "python app image-classification onnx"
cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e
cmr "python app image-classification onnx" --input=computer_mouse.jpg
+cmr "python app image-classification onnx" --input=computer_mouse.jpg --debug
cm find script "python app image-classification onnx"
@@ -46,7 +47,8 @@ cmr "install python-venv" --version_max=3.10.11 --name=mlperf
cmr "get ml-model stable-diffusion"
cmr "get ml-model huggingface zoo _model-stub.alpindale/Llama-2-13b-ONNX" --model_filename=FP32/LlamaV2_13B_float32.onnx --skip_cache
-cmr "get coco dataset _val _2014"
+cmr "get dataset coco _val _2014"
+cmr "get dataset openimages" -j
cm show cache
cm show cache "get ml-model stable-diffusion"
@@ -57,21 +59,34 @@ cmr "python app image-classification onnx" --input=computer_mouse.jpg
cm rm cache -f
cmr "python app image-classification onnx" --input=computer_mouse.jpg --adr.onnxruntime.version_max=1.16.0
-
+cmr "get cuda" --version_min=12.0.0 --version_max=12.3.1
cmr "python app image-classification onnx _cuda" --input=computer_mouse.jpg
-cmr "cm gui" --script="python app image-classification onnx"
+cm gui script "python app image-classification onnx"
cm docker script "python app image-classification onnx" --input=computer_mouse.jpg
cm docker script "python app image-classification onnx" --input=computer_mouse.jpg -j -docker_it
+cm docker script "get coco dataset _val _2017" --to=d:\Downloads\COCO-2017-val --store=d:\Downloads --docker_cm_repo=ctuning@mlcommons-ck
+
cmr "run common mlperf inference" --implementation=nvidia --model=bert-99 --category=datacenter --division=closed
cm find script "run common mlperf inference"
+cmr "get generic-python-lib _package.torch" --version=2.1.1
+cmr "get generic-python-lib _package.torchvision" --version=0.16.2
+cmr "python app image-classification torch" --input=computer_mouse.jpg
+
+
+cmr "install llvm prebuilt" --version=17.0.6
+cmr "app image corner-detection"
+
+cmr "get conda"
+
cm pull repo ctuning@cm-reproduce-research-projects
cmr "reproduce paper micro-2023 victima _install_deps"
cmr "reproduce paper micro-2023 victima _run"
+
```
```python
diff --git a/cm-mlops/automation/script/module.py b/cm-mlops/automation/script/module.py
index 4f81ece1ad..6a2ce81899 100644
--- a/cm-mlops/automation/script/module.py
+++ b/cm-mlops/automation/script/module.py
@@ -127,6 +127,7 @@ def run(self, i):
(save_env) (bool): if True, save env and state to tmp-env.sh/bat and tmp-state.json
(shell) (bool): if True, save env with cmd/bash and run it
+ (debug) (bool): the same as shell
(recursion) (bool): True if recursive call.
Useful when preparing the global bat file or Docker container
@@ -180,7 +181,7 @@ def run(self, i):
* state (dict): global state (updated by this script - includes new_state)
"""
-
+
from cmind import utils
import copy
import time
@@ -279,9 +280,14 @@ def run(self, i):
skip_cache = i.get('skip_cache', False)
force_cache = i.get('force_cache', False)
+
fake_run = i.get('fake_run', False)
fake_run = i.get('fake_run', False) if 'fake_run' in i else i.get('prepare', False)
+ if fake_run: env['CM_TMP_FAKE_RUN']='yes'
+
fake_deps = i.get('fake_deps', False)
+ if fake_deps: env['CM_TMP_FAKE_DEPS']='yes'
+
run_state = i.get('run_state', self.run_state)
if run_state.get('version_info', '') == '':
run_state['version_info'] = {}
@@ -289,6 +295,7 @@ def run(self, i):
run_state['parent'] = None
if fake_deps:
run_state['fake_deps'] = True
+
print_deps = i.get('print_deps', False)
print_readme = i.get('print_readme', False)
@@ -454,9 +461,10 @@ def run(self, i):
cm_script_info += '"'
- if verbose:
- print ('')
+# if verbose:
+# print ('')
+ print ('')
print (recursion_spaces + '* ' + cm_script_info)
@@ -1097,6 +1105,8 @@ def run(self, i):
if r['return']>0: return r
version = r['meta'].get('version')
+ print (recursion_spaces + ' ! load {}'.format(path_to_cached_state_file))
+
################################################################################################
# IF REUSE FROM CACHE - update env and state from cache!
@@ -1521,7 +1531,17 @@ def run(self, i):
utils.merge_dicts({'dict1':env, 'dict2':const, 'append_lists':True, 'append_unique':True})
utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True})
- r = detect_state_diff(env, saved_env, new_env_keys_from_meta, new_state_keys_from_meta, state, saved_state)
+ if i.get('force_new_env_keys', []):
+ new_env_keys = i['force_new_env_keys']
+ else:
+ new_env_keys = new_env_keys_from_meta
+
+ if i.get('force_new_state_keys', []):
+ new_state_keys = i['force_new_state_keys']
+ else:
+ new_state_keys = new_state_keys_from_meta
+
+ r = detect_state_diff(env, saved_env, new_env_keys, new_state_keys, state, saved_state)
if r['return']>0: return r
new_env = r['new_env']
@@ -1609,11 +1629,14 @@ def run(self, i):
script_path = os.getcwd()
os.chdir(current_path)
- if not i.get('dirty', False) and not cache:
+ shell = i.get('shell', False)
+ if not shell:
+ shell = i.get('debug', False)
+
+ if not shell and not i.get('dirty', False) and not cache:
clean_tmp_files(clean_files, recursion_spaces)
# Record new env and new state in the current dir if needed
- shell = i.get('shell', False)
if save_env or shell:
# Check if script_prefix in the state from other components
where_to_add = len(os_info['start_script'])
@@ -1625,13 +1648,15 @@ def run(self, i):
env_script.insert(where_to_add, x)
if shell:
- x = 'cmd' if os_info['platform'] == 'windows' else 'bash'
+ x=['cmd', '.', '','.bat',''] if os_info['platform'] == 'windows' else ['bash', ' ""', '"','.sh','. ./']
env_script.append('\n')
- env_script.append('echo "Working path: {}"'.format(script_path))
- env_script.append('echo "Running debug shell. Type exit to quit ..."\n')
+ env_script.append('echo{}\n'.format(x[1]))
+ env_script.append('echo {}Working path: {}{}'.format(x[2], script_path, x[2]))
+ env_script.append('echo {}Running debug shell. Change and run "tmp-run{}". Type exit to quit ...{}\n'.format(x[2],x[3],x[2]))
+ env_script.append('echo{}\n'.format(x[1]))
env_script.append('\n')
- env_script.append(x)
+ env_script.append(x[0])
env_file = self.tmp_file_env + bat_ext
@@ -2154,7 +2179,7 @@ def add(self, i):
console = i.get('out') == 'con'
- # Try to find experiment artifact by alias and/or tags
+ # Try to find script artifact by alias and/or tags
ii = utils.sub_input(i, self.cmind.cfg['artifact_keys'])
parsed_artifact = i.get('parsed_artifact',[])
@@ -2511,6 +2536,9 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a
if not d.get('tags'):
continue
+ if d.get('skip_if_fake_run', False) and env.get('CM_TMP_FAKE_RUN','')=='yes':
+ continue
+
if "enable_if_env" in d:
if not enable_or_skip_script(d["enable_if_env"], env):
continue
@@ -3408,7 +3436,7 @@ def get_default_path_list(self, i):
############################################################
def doc(self, i):
"""
- Add CM automation.
+ Document CM script.
Args:
(CM input dict):
@@ -3433,11 +3461,47 @@ def doc(self, i):
return utils.call_internal_module(self, __file__, 'module_misc', 'doc', i)
+ ############################################################
+ def gui(self, i):
+ """
+ Run GUI for CM script.
+
+ Args:
+ (CM input dict):
+
+ Returns:
+ (CM return dict):
+
+ * return (int): return code == 0 if no error and >0 if error
+ * (error) (str): error string if return>0
+
+ """
+
+ artifact = i.get('artifact', '')
+ tags = ''
+ if artifact != '':
+ if ' ' in artifact:
+ tags = artifact.replace(' ',',')
+
+ if tags=='':
+ tags = i.get('tags','')
+
+ if 'tags' in i:
+ del(i['tags'])
+
+ i['action']='run'
+ i['artifact']='gui'
+ i['parsed_artifact']=[('gui','605cac42514a4c69')]
+ i['script']=tags.replace(',',' ')
+
+ return self.cmind.access(i)
+
+
############################################################
def dockerfile(self, i):
"""
- Add CM automation.
+ Generate Dockerfile for CM script.
Args:
(CM input dict):
@@ -3465,7 +3529,7 @@ def dockerfile(self, i):
############################################################
def docker(self, i):
"""
- Add CM automation.
+ Run CM script in an automatically-generated container.
Args:
(CM input dict):
@@ -3935,6 +3999,10 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
print (recursion_spaces + ' - Running native script "{}" from temporal script "{}" in "{}" ...'.format(path_to_run_script, run_script, cur_dir))
print ('')
+ print (recursion_spaces + ' ! cd {}'.format(cur_dir))
+ print (recursion_spaces + ' ! call {} from {}'.format(run_script, path_to_run_script))
+
+
# Prepare env variables
import copy
script = copy.deepcopy(os_info['start_script'])
@@ -3949,12 +4017,14 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
# Check if run bash/cmd before running the command (for debugging)
if debug_script_tags !='' and all(item in found_script_tags for item in debug_script_tags.split(',')):
- x = 'cmd' if os_info['platform'] == 'windows' else 'bash'
+ x=['cmd', '.', '','.bat'] if os_info['platform'] == 'windows' else ['bash', ' ""', '"','.sh']
script.append('\n')
- script.append('echo "Running debug shell. Type exit to quit ..."\n')
+ script.append('echo{}\n'.format(x[1]))
+ script.append('echo {}Running debug shell. Type exit to resume script execution ...{}\n'.format(x[2],x[3],x[2]))
+ script.append('echo{}\n'.format(x[1]))
script.append('\n')
- script.append(x)
+ script.append(x[0])
# Append batch file to the tmp script
script.append('\n')
@@ -4013,6 +4083,11 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"):
utils.merge_dicts({'dict1':env, 'dict2':updated_env, 'append_lists':True, 'append_unique':True})
+
+ if customize_code is not None:
+ print (recursion_spaces+' ! call "{}" from {}'.format(postprocess, customize_code.__file__))
+
+
if len(posthook_deps)>0 and (postprocess == "postprocess"):
r = script_automation._call_run_deps(posthook_deps, local_env_keys, local_env_keys_from_meta, env, state, const, const_state,
add_deps_recursive, recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, run_state)
diff --git a/cm-mlops/automation/script/module_misc.py b/cm-mlops/automation/script/module_misc.py
index a0e55b8408..92da5d42d1 100644
--- a/cm-mlops/automation/script/module_misc.py
+++ b/cm-mlops/automation/script/module_misc.py
@@ -1051,6 +1051,47 @@ def doc(i):
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+def update_path_for_docker(path, mounts, force_path_target=''):
+
+ path_orig = ''
+ path_target = ''
+
+ if path!='': # and (os.path.isfile(path) or os.path.isdir(path)):
+ path = os.path.abspath(path)
+
+ path_target = path
+ path_orig = path
+
+ if os.name == 'nt':
+ from pathlib import PureWindowsPath, PurePosixPath
+
+ x = PureWindowsPath(path_orig)
+ path_target = str(PurePosixPath('/', *x.parts[1:]))
+
+ if not path_target.startswith('/'): path_target='/'+path_target
+
+ path_target='/cm-mount'+path_target if force_path_target=='' else force_path_target
+
+ # If file, mount directory
+ if os.path.isfile(path):
+ x = os.path.dirname(path_orig) + ':' + os.path.dirname(path_target)
+ else:
+ x = path_orig + ':' + path_target
+
+ # CHeck if no duplicates
+ to_add = True
+ for y in mounts:
+ if y.lower()==x.lower():
+ to_add = False
+ break
+
+ if to_add:
+ mounts.append(x)
+
+
+ return (path_orig, path_target)
+
############################################################
def process_inputs(i):
@@ -1063,34 +1104,6 @@ def process_inputs(i):
# Check if need to update/map/mount inputs and env
i_run_cmd = copy.deepcopy(i_run_cmd_arc)
- # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
- def update_path_for_docker(path, mounts):
-
- path_orig = ''
- path_target = ''
-
- if path!='' and (os.path.isfile(path) or os.path.isdir(path)):
- path = os.path.abspath(path)
-
- path_target = path
- path_orig = path
-
- if os.name == 'nt':
- from pathlib import PureWindowsPath, PurePosixPath
-
- x = PureWindowsPath(path_orig)
- path_target = str(PurePosixPath('/', *x.parts[1:]))
-
- if not path_target.startswith('/'): path_target='/'+path_target
- path_target='/cm-mount'+path_target
-
- # If file, mount directory
- if os.path.isfile(path):
- mounts.append(os.path.dirname(path_orig) + ':' + os.path.dirname(path_target))
- else:
- mounts.append(path_orig + ':' + path_target)
-
- return (path_orig, path_target)
def get_value_using_key_with_dots(d, k):
v = None
@@ -1570,6 +1583,11 @@ def docker(i):
i_run_cmd = r['run_cmd']
+ # Check if need to mount home directory
+ current_path_target = '/cm-mount/current'
+ if docker_settings.get('mount_current_dir','')=='yes':
+ update_path_for_docker('.', mounts, force_path_target=current_path_target)
+
_os = i.get('docker_os', meta.get('docker_os', 'ubuntu'))
version = i.get('docker_os_version', meta.get('docker_os_version', '22.04'))
@@ -1680,7 +1698,10 @@ def docker(i):
if r['return']>0: return r
run_cmd = r['run_cmd_string']
-
+
+ if docker_settings.get('mount_current_dir','')=='yes':
+ run_cmd = 'cd '+current_path_target+' && '+run_cmd
+
print ('')
print ('CM command line regenerated to be used inside Docker:')
print ('')
diff --git a/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml b/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml
index c3f869081e..b3fbb7bf93 100644
--- a/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml
+++ b/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml
@@ -65,9 +65,15 @@ deps:
variations:
cuda:
+ group: target
env:
USE_CUDA: yes
+ cpu:
+ group: target
+ default: yes
+ env:
+ USE_CPU: yes
input_mapping:
input: CM_IMAGE
diff --git a/cm-mlops/script/app-image-classification-onnx-py/customize.py b/cm-mlops/script/app-image-classification-onnx-py/customize.py
index e1c07dc2e3..43098c71f7 100644
--- a/cm-mlops/script/app-image-classification-onnx-py/customize.py
+++ b/cm-mlops/script/app-image-classification-onnx-py/customize.py
@@ -53,6 +53,12 @@ def postprocess(i):
except Exception as e:
print ('CM warning: {}'.format(e))
+ top_classification = data.get('top_classification','')
+
+ if top_classification!='':
+ print ('')
+ x = 'Top classification: {}'.format(top_classification)
+ print ('='*len(x))
+ print (x)
-
return {'return':0}
diff --git a/cm-mlops/script/app-image-classification-torch-py/README-extra.md b/cm-mlops/script/app-image-classification-torch-py/README-extra.md
index b9eb292733..6628885061 100644
--- a/cm-mlops/script/app-image-classification-torch-py/README-extra.md
+++ b/cm-mlops/script/app-image-classification-torch-py/README-extra.md
@@ -1,12 +1,16 @@
-# CUDA version
+# CPU
+
+## 20240129; Windows 11
```bash
-cm run script "install python-venv" --name=test
-cm run script "python app image-classification pytorch _cuda"
-cm run script "python app image-classification pytorch _cuda" --input=src/computer_mouse.jpg
+cmr "get generic-python-lib _package.torch" --version=2.1.1
+cmr "get generic-python-lib _package.torchvision" --version=0.16.2
```
-## Tested on Windows 10
-* ONNX Runtime 1.13.1 with CUDA
-* CUDA 11.6
-* cuDNN 8.5.0.96
+# CUDA
+
+```bash
+cm run script "install python-venv" --name=test
+cm run script "python app image-classification pytorch _cuda" --adr.python.name=test
+cm run script "python app image-classification pytorch _cuda" --adr.python.name=test --input=src/computer_mouse.jpg
+```
diff --git a/cm-mlops/script/app-image-corner-detection/README-extra.md b/cm-mlops/script/app-image-corner-detection/README-extra.md
index 8bcff69f95..6161c6521c 100644
--- a/cm-mlops/script/app-image-corner-detection/README-extra.md
+++ b/cm-mlops/script/app-image-corner-detection/README-extra.md
@@ -6,3 +6,9 @@ cm run script "app image corner-detection" -add_deps_recursive.compiler.tags=llv
cm run script "app image corner-detection" -add_deps_recursive.compiler.tags=gcc
cm run script "app image corner-detection" -add_deps_recursive.compiler.tags=llvm --add_deps_recursive.compiler.version_min=11.0.0 --add_deps_recursive.compiler.version_max=13.0.0
```
+
+## Reproducibility matrix
+
+* Ubuntu 22.04; x64; LLVM 17.06
+* Windows 11; x64; LLVM 17.06
+
diff --git a/cm-mlops/script/app-mlperf-inference-reference/customize.py b/cm-mlops/script/app-mlperf-inference-reference/customize.py
index df880c219d..c22344df03 100644
--- a/cm-mlops/script/app-mlperf-inference-reference/customize.py
+++ b/cm-mlops/script/app-mlperf-inference-reference/customize.py
@@ -66,10 +66,11 @@ def preprocess(i):
env['CM_MLPERF_CONF'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "mlperf.conf")
+ x="" if os_info['platform'] == 'windows' else "'"
if "llama2-70b" in env['CM_MODEL']:
- env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf-conf '" + env['CM_MLPERF_CONF'] + "'"
+ env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf-conf " + x+ env['CM_MLPERF_CONF'] + x
else:
- env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf_conf '" + env['CM_MLPERF_CONF'] + "'"
+ env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf_conf "+ x + env['CM_MLPERF_CONF'] + x
env['MODEL_DIR'] = env.get('CM_ML_MODEL_PATH')
if not env['MODEL_DIR']:
@@ -95,10 +96,11 @@ def preprocess(i):
ml_model_name = env['CM_MODEL']
if 'CM_MLPERF_USER_CONF' in env:
user_conf_path = env['CM_MLPERF_USER_CONF']
+ x="" if os_info['platform'] == 'windows' else "'"
if 'llama2-70b' in env['CM_MODEL']:
- scenario_extra_options += " --user-conf '" + user_conf_path + "'"
+ scenario_extra_options += " --user-conf " + x + user_conf_path + x
else:
- scenario_extra_options += " --user_conf '" + user_conf_path + "'"
+ scenario_extra_options += " --user_conf " + x + user_conf_path + x
mode = env['CM_MLPERF_LOADGEN_MODE']
mode_extra_options = ""
@@ -139,7 +141,7 @@ def preprocess(i):
env['CM_MLPERF_OUTPUT_DIR'] = os.getcwd()
mlperf_implementation = env.get('CM_MLPERF_IMPLEMENTATION', 'reference')
- cmd, run_dir = get_run_cmd(env, scenario_extra_options, mode_extra_options, dataset_options, mlperf_implementation)
+ cmd, run_dir = get_run_cmd(os_info, env, scenario_extra_options, mode_extra_options, dataset_options, mlperf_implementation)
if env.get('CM_NETWORK_LOADGEN', '') == "lon":
@@ -158,15 +160,15 @@ def preprocess(i):
return {'return':0}
-def get_run_cmd(env, scenario_extra_options, mode_extra_options, dataset_options, implementation="reference"):
+def get_run_cmd(os_info, env, scenario_extra_options, mode_extra_options, dataset_options, implementation="reference"):
if implementation == "reference":
- return get_run_cmd_reference(env, scenario_extra_options, mode_extra_options, dataset_options)
+ return get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_options, dataset_options)
if implementation == "nvidia":
- return get_run_cmd_nvidia(env, scenario_extra_options, mode_extra_options, dataset_options)
+ return get_run_cmd_nvidia(os_info, env, scenario_extra_options, mode_extra_options, dataset_options)
return "", os.getcwd()
-def get_run_cmd_reference(env, scenario_extra_options, mode_extra_options, dataset_options):
+def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_options, dataset_options):
if env['CM_MODEL'] in [ "gptj-99", "gptj-99.9" ]:
@@ -190,9 +192,15 @@ def get_run_cmd_reference(env, scenario_extra_options, mode_extra_options, datas
env['RUN_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "vision", "classification_and_detection")
env['OUTPUT_DIR'] = env['CM_MLPERF_OUTPUT_DIR']
if env.get('CM_MLPERF_VISION_DATASET_OPTION','') == '' and env.get('CM_MLPERF_DEVICE') != "tpu":
- cmd = "./run_local.sh " + env['CM_MLPERF_BACKEND'] + ' ' + \
- env['CM_MODEL'] + ' ' + env['CM_MLPERF_DEVICE'] + " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + " " + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \
- scenario_extra_options + mode_extra_options + dataset_options
+ if os_info['platform'] == 'windows':
+ cmd = "python python/main.py --profile "+env['CM_MODEL']+"-"+env['CM_MLPERF_BACKEND'] + \
+ " --model=" + env['CM_ML_MODEL_FILE_WITH_PATH'] + ' --dataset-path=' + env['CM_DATASET_PREPROCESSED_PATH'] + \
+ " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + " " + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \
+ scenario_extra_options + mode_extra_options + dataset_options
+ else:
+ cmd = "./run_local.sh " + env['CM_MLPERF_BACKEND'] + ' ' + \
+ env['CM_MODEL'] + ' ' + env['CM_MLPERF_DEVICE'] + " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + " " + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \
+ scenario_extra_options + mode_extra_options + dataset_options
return cmd, env['RUN_DIR']
if env['CM_MLPERF_BACKEND'] == "ncnn":
diff --git a/cm-mlops/script/benchmark-program-mlperf/customize.py b/cm-mlops/script/benchmark-program-mlperf/customize.py
index e08823fe5a..a333b6c078 100644
--- a/cm-mlops/script/benchmark-program-mlperf/customize.py
+++ b/cm-mlops/script/benchmark-program-mlperf/customize.py
@@ -9,9 +9,16 @@ def preprocess(i):
def postprocess(i):
+ os_info = i['os_info']
env = i['env']
+
env['CM_MLPERF_RUN_CMD'] = env.get('CM_RUN_CMD')
+
+
if env.get('CM_MLPERF_POWER', '') == "yes":
+ if os_info['platform'] == 'windows':
+ return {'return':1, 'error':'TBD: this script is not yet supported on Windows'}
+
if env.get('CM_MLPERF_SHORT_RANGING_RUN', '') != 'no':
os.system("echo '0' > "+env.get('CM_RUN_DIR','')+ "/count.txt")
env['CM_MLPERF_RUN_CMD'] = "CM_MLPERF_RUN_COUNT=\$(cat \${CM_RUN_DIR}/count.txt); echo \${CM_MLPERF_RUN_COUNT}; CM_MLPERF_RUN_COUNT=\$((CM_MLPERF_RUN_COUNT+1)); echo \${CM_MLPERF_RUN_COUNT} > \${CM_RUN_DIR}/count.txt && if [ \${CM_MLPERF_RUN_COUNT} -eq \'1\' ]; then export CM_MLPERF_USER_CONF=\${CM_MLPERF_RANGING_USER_CONF}; else export CM_MLPERF_USER_CONF=\${CM_MLPERF_TESTING_USER_CONF}; fi && "+env.get('CM_RUN_CMD','').strip()
diff --git a/cm-mlops/script/benchmark-program/run.bat b/cm-mlops/script/benchmark-program/run.bat
index e00f449760..62929b0e8e 100644
--- a/cm-mlops/script/benchmark-program/run.bat
+++ b/cm-mlops/script/benchmark-program/run.bat
@@ -2,9 +2,6 @@
cd %CM_RUN_DIR%
-echo %CM_RUN_CMD%
-pause
-
%CM_RUN_CMD%
IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
diff --git a/cm-mlops/script/get-conda/_cm.json b/cm-mlops/script/get-conda/_cm.json
index eef50fa6cf..3253a3ff56 100644
--- a/cm-mlops/script/get-conda/_cm.json
+++ b/cm-mlops/script/get-conda/_cm.json
@@ -29,11 +29,15 @@
"name.#": {
"env": {
"CM_CONDA_PREFIX_NAME": "#"
+ },
+ "adr": {
+ "conda-package": {
+ "tags": "_name.#"
+ }
}
},
"python-3.8": {
"group": "conda-python",
- "default": true,
"env": {
"CM_CONDA_PYTHON_VERSION": "3.8"
}
diff --git a/cm-mlops/script/get-conda/customize.py b/cm-mlops/script/get-conda/customize.py
index d5bd0ef439..d8ef13e343 100644
--- a/cm-mlops/script/get-conda/customize.py
+++ b/cm-mlops/script/get-conda/customize.py
@@ -13,27 +13,29 @@ def preprocess(i):
conda_prefix_name = env.get('CM_CONDA_PREFIX_NAME', '')
r = None
+ file_name = 'conda.exe' if os_info['platform'] == 'windows' else 'conda'
if conda_prefix_name == '':
- file_name = 'conda.exe' if os_info['platform'] == 'windows' else 'conda'
tmp_path = env.get('CM_CONDA_INSTALL_PATH', env.get('CM_TMP_PATH', ''))
if tmp_path:
- tmp_path+=":"
+ x = ';' if os_info['platform'] == 'windows' else ':'
+ tmp_path+=x
conda_path = os.path.join(os.path.expanduser("~"), "miniconda3", "bin")
if os.path.exists(conda_path):
tmp_path += os.path.join(os.path.expanduser("~"), "miniconda3", "bin")
env['CM_TMP_PATH'] = tmp_path
r = i['automation'].find_artifact({'file_name': file_name,
- 'env': env,
- 'os_info':os_info,
- 'default_path_env_key': 'PATH',
- 'detect_version':True,
- 'env_path_key':'CM_CONDA_BIN_WITH_PATH',
- 'run_script_input':i['run_script_input'],
- 'recursion_spaces':recursion_spaces})
+ 'env': env,
+ 'os_info':os_info,
+ 'default_path_env_key': 'PATH',
+ 'detect_version':True,
+ 'env_path_key':'CM_CONDA_BIN_WITH_PATH',
+ 'run_script_input':i['run_script_input'],
+ 'recursion_spaces':recursion_spaces})
else:
env['CM_CONDA_INSTALL_PATH'] = os.path.join(os.getcwd(), "miniconda3")
- env['CM_CONDA_BIN_WITH_PATH'] = os.path.join(env['CM_CONDA_INSTALL_PATH'], "bin", "conda")
+ bin_dir = 'Scripts' if os_info['platform'] == 'windows' else 'bin'
+ env['CM_CONDA_BIN_WITH_PATH'] = os.path.join(env['CM_CONDA_INSTALL_PATH'], bin_dir, file_name)
if conda_prefix_name != '' or r['return'] >0 :
if conda_prefix_name != '' or r['return'] == 16:
@@ -47,6 +49,12 @@ def preprocess(i):
r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'install'})
if r['return']>0: return r
+ # Grigori: temporal fix - should be generalized/improved above
+ if os_info['platform'] == 'windows' and env.get('CM_CONDA_BIN_WITH_PATH','')=='':
+ env['CM_CONDA_INSTALL_PATH'] = os.path.join(os.getcwd(), "miniconda3")
+ env['CM_CONDA_BIN_WITH_PATH'] = os.path.join(env['CM_CONDA_INSTALL_PATH'], 'Scripts', file_name)
+
+
else:
found_path = r['found_path']
env['+PATH'] = [ found_path ]
diff --git a/cm-mlops/script/get-conda/install.bat b/cm-mlops/script/get-conda/install.bat
new file mode 100644
index 0000000000..2528840d9c
--- /dev/null
+++ b/cm-mlops/script/get-conda/install.bat
@@ -0,0 +1,9 @@
+if exist Miniconda3-latest-Windows-x86_64.exe (
+ del /Q /S Miniconda3-latest-Windows-x86_64.exe
+)
+
+wget --no-check-certificate https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe
+IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
+
+start /wait "" Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /D=%CD%\miniconda3
+IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
diff --git a/cm-mlops/script/get-conda/run.bat b/cm-mlops/script/get-conda/run.bat
new file mode 100644
index 0000000000..99b9d97d27
--- /dev/null
+++ b/cm-mlops/script/get-conda/run.bat
@@ -0,0 +1 @@
+%CM_CONDA_BIN_WITH_PATH% --version > tmp-ver.out
diff --git a/cm-mlops/script/get-dataset-cnndm/_cm.json b/cm-mlops/script/get-dataset-cnndm/_cm.json
index fd70b9d6b9..36af72cdc5 100644
--- a/cm-mlops/script/get-dataset-cnndm/_cm.json
+++ b/cm-mlops/script/get-dataset-cnndm/_cm.json
@@ -16,16 +16,17 @@
"python",
"python3"
],
- "tags": "get,python3"
+ "tags": "get,python3",
+ "version_max": "3.9.999"
},
{
- "force_env_keys": [
- "CM_GIT_*"
- ],
"names": [
"inference-src"
],
- "tags": "mlperf,inference,source"
+ "tags": "mlperf,inference,source",
+ "skip_if_env": {
+ "CM_CNNDM_INTEL_VARIATION": [ "yes" ]
+ }
},
{
"tags": "get,generic-python-lib,_package.simplejson"
@@ -43,11 +44,6 @@
"env": {
"CM_DATASET": "CNNDM"
},
- "new_env_keys": [
- "CM_DATASET_PATH",
- "CM_DATASET_EVAL_PATH",
- "CM_CALIBRATION_DATASET_PATH"
- ],
"tags": [
"get",
"dataset",
@@ -59,17 +55,33 @@
"uid": "aed298c156e24257",
"variations": {
"calibration": {
+ "new_env_keys": [
+ "CM_CALIBRATION_DATASET_PATH",
+ "CM_CALIBRATION_DATASET_CNNDM_PATH"
+ ],
"env": {
"CM_DATASET_CALIBRATION": "yes"
},
"group": "dataset-type"
},
"validation": {
+ "new_env_keys": [
+ "CM_DATASET_PATH",
+ "CM_DATASET_EVAL_PATH",
+ "CM_DATASET_CNNDM_EVAL_PATH"
+ ],
"default": true,
"env": {
"CM_DATASET_CALIBRATION": "no"
},
"group": "dataset-type"
+ },
+ "intel": {
+ },
+ "intel,validation": {
+ "env": {
+ "CM_CNNDM_INTEL_VARIATION": "yes"
+ }
}
}
}
diff --git a/cm-mlops/script/get-dataset-cnndm/customize.py b/cm-mlops/script/get-dataset-cnndm/customize.py
index d7ca59f20c..27363d8000 100644
--- a/cm-mlops/script/get-dataset-cnndm/customize.py
+++ b/cm-mlops/script/get-dataset-cnndm/customize.py
@@ -6,7 +6,10 @@ def preprocess(i):
env = i['env']
- print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'")
+ if env.get('CM_CNNDM_INTEL_VARIATION', '') == 'yes':
+ i['run_script_input']['script_name'] = "run-intel"
+ else:
+ print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'")
return {'return': 0}
@@ -16,7 +19,11 @@ def postprocess(i):
if env.get('CM_DATASET_CALIBRATION','') == "no":
env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install')
env['CM_DATASET_EVAL_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_eval.json')
+ env['CM_DATASET_CNNDM_EVAL_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_eval.json')
+ env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_DATASET_PATH']
else:
env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_dailymail_calibration.json')
+ env['CM_CALIBRATION_DATASET_CNNDM_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_dailymail_calibration.json')
+ env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_CALIBRATION_DATASET_PATH']
return {'return': 0}
diff --git a/cm-mlops/script/get-dataset-cnndm/run-intel.sh b/cm-mlops/script/get-dataset-cnndm/run-intel.sh
new file mode 100644
index 0000000000..067f158a56
--- /dev/null
+++ b/cm-mlops/script/get-dataset-cnndm/run-intel.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+CUR=${PWD}
+rm -rf install
+mkdir -p install
+
+export DATASET_CNNDM_PATH=${CUR}/install
+
+wget -nc https://raw.githubusercontent.com/mlcommons/inference_results_v3.1/main/closed/Intel/code/gptj-99/pytorch-cpu/download-dataset.py
+test $? -eq 0 || exit 1
+
+cmd="${CM_PYTHON_BIN_WITH_PATH} download-dataset.py --split validation --output-dir ${DATASET_CNNDM_PATH}"
+echo "$cmd"
+eval "$cmd"
+test $? -eq 0 || exit 1
diff --git a/cm-mlops/script/get-dataset-coco/README-extra.md b/cm-mlops/script/get-dataset-coco/README-extra.md
index 5dfa1885bc..9f19d2e8d7 100644
--- a/cm-mlops/script/get-dataset-coco/README-extra.md
+++ b/cm-mlops/script/get-dataset-coco/README-extra.md
@@ -92,3 +92,4 @@ It is useful when all files are already downloaded and saved for common use.
```bash
cmr "get coco dataset _val _2017" --to=d:\Downloads\COCO-2017-val --store=d:\Downloads
+```
diff --git a/cm-mlops/script/get-dataset-coco/_cm.json b/cm-mlops/script/get-dataset-coco/_cm.json
index 883e0b5363..a61234ad0c 100644
--- a/cm-mlops/script/get-dataset-coco/_cm.json
+++ b/cm-mlops/script/get-dataset-coco/_cm.json
@@ -17,7 +17,8 @@
"input_mapping": {
"from": "CM_FROM",
"store": "CM_STORE",
- "to": "CM_TO"
+ "to": "CM_TO",
+ "home": "CM_HOME_DIR"
},
"new_env_keys": [
"CM_DATASET_COCO*",
@@ -36,6 +37,7 @@
"get-dataset-coco-data",
"746e5dad5e784ad6"
],
+ "skip_if_fake_run": true,
"skip_if_env": {
"CM_DATASET_COCO_DETECTED": [
"yes"
@@ -61,6 +63,7 @@
"get-dataset-coco-annotations",
"edb6cd092ff64171"
],
+ "skip_if_fake_run": true,
"skip_if_env": {
"CM_DATASET_COCO_DETECTED": [
"yes"
@@ -116,5 +119,19 @@
},
"group": "type"
}
+ },
+ "docker": {
+ "skip_run_cmd":"no",
+ "mount_current_dir":"yes",
+ "input_paths": [
+ "store",
+ "from",
+ "to"
+ ],
+ "skip_input_for_fake_run": [
+ "store",
+ "from",
+ "to"
+ ]
}
}
diff --git a/cm-mlops/script/get-dataset-openimages/README.extra.md b/cm-mlops/script/get-dataset-openimages/README.extra.md
new file mode 100644
index 0000000000..b6f5d0812b
--- /dev/null
+++ b/cm-mlops/script/get-dataset-openimages/README.extra.md
@@ -0,0 +1,2 @@
+# Ubuntu 22.04
+`sudo apt-get install -y libgl1-mesa-dev`
diff --git a/cm-mlops/script/get-dataset-openimages/customize.py b/cm-mlops/script/get-dataset-openimages/customize.py
index 03bab05356..3040fff2ba 100644
--- a/cm-mlops/script/get-dataset-openimages/customize.py
+++ b/cm-mlops/script/get-dataset-openimages/customize.py
@@ -4,14 +4,64 @@
def preprocess(i):
+ os_info = i['os_info']
env = i['env']
- print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'")
+ print ("")
+ print ("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'")
+ print ("")
+
+ if os_info['platform'] == 'windows':
+ MLPERF_CLASSES=['Airplane','Antelope','Apple','Backpack','Balloon','Banana',
+ 'Barrel','Baseball bat','Baseball glove','Bee','Beer','Bench','Bicycle',
+ 'Bicycle helmet','Bicycle wheel','Billboard','Book','Bookcase','Boot',
+ 'Bottle','Bowl','Bowling equipment','Box','Boy','Brassiere','Bread',
+ 'Broccoli','Bronze sculpture','Bull','Bus','Bust','Butterfly','Cabinetry',
+ 'Cake','Camel','Camera','Candle','Candy','Cannon','Canoe','Carrot','Cart',
+ 'Castle','Cat','Cattle','Cello','Chair','Cheese','Chest of drawers','Chicken',
+ 'Christmas tree','Coat','Cocktail','Coffee','Coffee cup','Coffee table','Coin',
+ 'Common sunflower','Computer keyboard','Computer monitor','Convenience store',
+ 'Cookie','Countertop','Cowboy hat','Crab','Crocodile','Cucumber','Cupboard',
+ 'Curtain','Deer','Desk','Dinosaur','Dog','Doll','Dolphin','Door','Dragonfly',
+ 'Drawer','Dress','Drum','Duck','Eagle','Earrings','Egg (Food)','Elephant',
+ 'Falcon','Fedora','Flag','Flowerpot','Football','Football helmet','Fork',
+ 'Fountain','French fries','French horn','Frog','Giraffe','Girl','Glasses',
+ 'Goat','Goggles','Goldfish','Gondola','Goose','Grape','Grapefruit','Guitar',
+ 'Hamburger','Handbag','Harbor seal','Headphones','Helicopter','High heels',
+ 'Hiking equipment','Horse','House','Houseplant','Human arm','Human beard',
+ 'Human body','Human ear','Human eye','Human face','Human foot','Human hair',
+ 'Human hand','Human head','Human leg','Human mouth','Human nose','Ice cream',
+ 'Jacket','Jeans','Jellyfish','Juice','Kitchen & dining room table','Kite',
+ 'Lamp','Lantern','Laptop','Lavender (Plant)','Lemon','Light bulb','Lighthouse',
+ 'Lily','Lion','Lipstick','Lizard','Man','Maple','Microphone','Mirror',
+ 'Mixing bowl','Mobile phone','Monkey','Motorcycle','Muffin','Mug','Mule',
+ 'Mushroom','Musical keyboard','Necklace','Nightstand','Office building',
+ 'Orange','Owl','Oyster','Paddle','Palm tree','Parachute','Parrot','Pen',
+ 'Penguin','Personal flotation device','Piano','Picture frame','Pig','Pillow',
+ 'Pizza','Plate','Platter','Porch','Poster','Pumpkin','Rabbit','Rifle',
+ 'Roller skates','Rose','Salad','Sandal','Saucer','Saxophone','Scarf','Sea lion',
+ 'Sea turtle','Sheep','Shelf','Shirt','Shorts','Shrimp','Sink','Skateboard',
+ 'Ski','Skull','Skyscraper','Snake','Sock','Sofa bed','Sparrow','Spider','Spoon',
+ 'Sports uniform','Squirrel','Stairs','Stool','Strawberry','Street light',
+ 'Studio couch','Suit','Sun hat','Sunglasses','Surfboard','Sushi','Swan',
+ 'Swimming pool','Swimwear','Tank','Tap','Taxi','Tea','Teddy bear','Television',
+ 'Tent','Tie','Tiger','Tin can','Tire','Toilet','Tomato','Tortoise','Tower',
+ 'Traffic light','Train','Tripod','Truck','Trumpet','Umbrella','Van','Vase',
+ 'Vehicle registration plate','Violin','Wall clock','Waste container','Watch',
+ 'Whale','Wheel','Wheelchair','Whiteboard','Window','Wine','Wine glass','Woman',
+ 'Zebra','Zucchini']
+
+ x = ''
+ for v in MLPERF_CLASSES:
+ if x!='': x+=' '
+ x+='"'+v+'"'
+ env['CM_DATASET_OPENIMAGES_CLASSES']=x
return {'return': 0}
def postprocess(i):
env = i['env']
+
env['CM_DATASET_ANNOTATIONS_DIR_PATH'] = os.path.join(os.getcwd(), 'install', 'annotations')
if env.get('CM_DATASET_CALIBRATION','') == "no":
diff --git a/cm-mlops/script/get-dataset-openimages/run.bat b/cm-mlops/script/get-dataset-openimages/run.bat
index 46ee3ebd6d..3b1b6a15e8 100644
--- a/cm-mlops/script/get-dataset-openimages/run.bat
+++ b/cm-mlops/script/get-dataset-openimages/run.bat
@@ -15,5 +15,8 @@ if not "%CM_DATASET_SIZE%" == "" (
set MAX_IMAGES=
)
-%CM_PYTHON_BIN% tools\openimages.py %MAX_IMAGES% --dataset-dir=%INSTALL_DIR% --output-labels=openimages-mlperf.json
+%CM_PYTHON_BIN% tools\openimages.py %MAX_IMAGES% --dataset-dir=%INSTALL_DIR% --output-labels=openimages-mlperf.json --classes %CM_DATASET_OPENIMAGES_CLASSES%
IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%
+
+cd %INSTALL_DIR%
+move validation\data\* .
\ No newline at end of file
diff --git a/cm-mlops/script/get-generic-python-lib/_cm.json b/cm-mlops/script/get-generic-python-lib/_cm.json
index 7f1909d214..88ab4ce47c 100644
--- a/cm-mlops/script/get-generic-python-lib/_cm.json
+++ b/cm-mlops/script/get-generic-python-lib/_cm.json
@@ -73,6 +73,16 @@
"CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL": "yes"
}
},
+ "conda.#": {
+ "ad": {
+ "python-pip": {
+ "tags": "_conda.#"
+ },
+ "python3": {
+ "tags": "_conda.#"
+ }
+ }
+ },
"custom-python": {
"env": {
"CM_TMP_USE_CUSTOM_PYTHON": "on"
diff --git a/cm-mlops/script/get-generic-python-lib/customize.py b/cm-mlops/script/get-generic-python-lib/customize.py
index 5027d82376..c0d5b4334f 100644
--- a/cm-mlops/script/get-generic-python-lib/customize.py
+++ b/cm-mlops/script/get-generic-python-lib/customize.py
@@ -16,10 +16,12 @@ def preprocess(i):
if package_name == "onnxruntime_gpu":
# https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements
- cuda_version_split = env['CM_CUDA_VERSION'].split(".")
- if int(cuda_version_split[0]) >= 12:
- # env['CM_INSTALL_ONNXRUNTIME_GPU_FROM_SRC'] = "yes"
- return {'return': 1, 'error':'at this moment, onnxruntime_gpu needs CUDA < 12'}
+ cuda_version = env.get('CM_CUDA_VERSION','').strip()
+ if cuda_version!='':
+ cuda_version_split = cuda_version.split('.')
+ if int(cuda_version_split[0]) >= 12:
+ # env['CM_INSTALL_ONNXRUNTIME_GPU_FROM_SRC'] = "yes"
+ return {'return': 1, 'error':'at this moment, PIP package "onnxruntime_gpu" needs CUDA < 12'}
extra = env.get('CM_GENERIC_PYTHON_PIP_EXTRA','')
if (pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23) and ('--break-system-packages' not in extra):
diff --git a/cm-mlops/script/get-generic-python-lib/run.sh b/cm-mlops/script/get-generic-python-lib/run.sh
index 0d3ea80052..5a2a4b56bb 100644
--- a/cm-mlops/script/get-generic-python-lib/run.sh
+++ b/cm-mlops/script/get-generic-python-lib/run.sh
@@ -3,5 +3,5 @@
CM_TMP_CURRENT_SCRIPT_PATH=${CM_TMP_CURRENT_SCRIPT_PATH:-$PWD}
cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/detect-version.py > tmp-ver.out 2> tmp-ver.err"
-echo $cmd
+#echo $cmd
eval $cmd
diff --git a/cm-mlops/script/get-ml-model-gptj/_cm.json b/cm-mlops/script/get-ml-model-gptj/_cm.json
index f5be804c33..51f6cf3bb1 100644
--- a/cm-mlops/script/get-ml-model-gptj/_cm.json
+++ b/cm-mlops/script/get-ml-model-gptj/_cm.json
@@ -53,7 +53,6 @@
}
},
"fp32": {
- "default": true,
"env": {
"CM_ML_MODEL_INPUT_DATA_TYPES": "fp32",
"CM_ML_MODEL_PRECISION": "fp32",
@@ -92,6 +91,13 @@
}
}
},
+ "int4": {
+ "env": {
+ "CM_ML_MODEL_INPUT_DATA_TYPES": "int4",
+ "CM_ML_MODEL_WEIGHT_DATA_TYPES": "int4"
+ },
+ "group": "precision"
+ },
"uint8": {
"env": {
"CM_ML_MODEL_INPUT_DATA_TYPES": "uint8",
@@ -99,6 +105,78 @@
"CM_ML_MODEL_WEIGHT_DATA_TYPES": "uint8"
},
"group": "precision"
+ },
+ "mlcommons": {
+ "default_variations": {
+ "precision": "fp32"
+ },
+ "group": "model-provider",
+ "default": true
+ },
+ "intel": {
+ "default_variations": {
+ "framework": "pytorch"
+ },
+ "group": "model-provider"
+ },
+ "pytorch,intel": {
+ "default_variations": {
+ "precision": "int8"
+ },
+ "adr": {
+ "conda-package": {
+ "tags": "_name.gptj-pt"
+ }
+ },
+ "deps": [
+ {
+ "tags": "get,mlperf,inference,results",
+ "version": "v3.1"
+ },
+ {
+ "tags": "get,ml-model,gpt-j,_fp32,_pytorch",
+ "env": {
+ "CM_GPTJ_INTEL_MODEL": ""
+ },
+ "force_new_env_keys": [
+ "GPTJ_CHECKPOINT_PATH"
+ ]
+ },
+ {
+ "tags": "get,conda,_name.gptj-pt"
+ },
+ {
+ "tags": "get,python,_conda.gptj-pt"
+ },
+ {
+ "names": [
+ "conda-package",
+ "intel-openmp"
+ ],
+ "tags": "get,generic,conda-package,_package.intel-openmp,_source.intel",
+ "version": "2023.1.0"
+ },
+ {
+ "tags": "get,generic,conda-package,_package.jemalloc,_source.conda-forge",
+ "names": [
+ "conda-package",
+ "jemalloc"
+ ]
+ },
+ {
+ "tags": "install,ipex,from.src,_for-intel-mlperf-inference-v3.1-gptj"
+ },
+ {
+ "tags": "get,dataset,cnndm,_calibration"
+ }
+ ],
+ "env": {
+ "CM_GPTJ_INTEL_MODEL": "yes"
+ }
+ },
+ "pytorch,int8,intel": {
+ },
+ "pytorch,int4,intel": {
}
}
}
diff --git a/cm-mlops/script/get-ml-model-gptj/customize.py b/cm-mlops/script/get-ml-model-gptj/customize.py
index 1b1b7dd249..36655d6c34 100644
--- a/cm-mlops/script/get-ml-model-gptj/customize.py
+++ b/cm-mlops/script/get-ml-model-gptj/customize.py
@@ -6,10 +6,26 @@ def preprocess(i):
os_info = i['os_info']
env = i['env']
- path = env.get('GPTJ_CHECKPOINT_PATH', '').strip()
+ if env.get('CM_GPTJ_INTEL_MODEL', '') == 'yes':
+ i['run_script_input']['script_name'] = 'run-intel'
+ harness_root = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'code', 'gptj-99', 'pytorch-cpu')
+ print(f"Harness Root: {harness_root}")
+ env['CM_HARNESS_CODE_ROOT'] = harness_root
+ env['CM_CALIBRATION_CODE_ROOT'] = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'calibration')
- if path == '' or not os.path.exists(path):
- env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes'
+ env['CHECKPOINT_DIR'] = env['GPTJ_CHECKPOINT_PATH']
+
+ env['QUANTIZED_MODEL_DIR'] = os.getcwd()
+
+ if env['CM_ML_MODEL_WEIGHT_DATA_TYPES'] == "int8":
+ env['INT8_MODEL_DIR'] = os.getcwd()
+ else:
+ env['INT4_MODEL_DIR'] = os.getcwd()
+ else:
+ path = env.get('GPTJ_CHECKPOINT_PATH', '').strip()
+
+ if path == '' or not os.path.exists(path):
+ env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes'
return {'return':0}
@@ -17,7 +33,9 @@ def postprocess(i):
env = i['env']
- env['GPTJ_CHECKPOINT_PATH'] = os.path.join(env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")
+ if os.path.exists(os.path.join(env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")):
+ env['GPTJ_CHECKPOINT_PATH'] = os.path.join(env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")
+
env['CM_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_CHECKPOINT_PATH']
env['CM_ML_MODEL_FILE'] = os.path.basename(env['CM_ML_MODEL_FILE_WITH_PATH'])
env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_FILE_WITH_PATH']
diff --git a/cm-mlops/script/get-ml-model-gptj/run-int4-calibration.sh b/cm-mlops/script/get-ml-model-gptj/run-int4-calibration.sh
new file mode 100644
index 0000000000..45c3669e56
--- /dev/null
+++ b/cm-mlops/script/get-ml-model-gptj/run-int4-calibration.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export PATH=${CM_CONDA_BIN_PATH}:$PATH
+
+echo ${CM_CALIBRATION_CODE_ROOT}
+cd ${CM_CALIBRATION_CODE_ROOT}/gpt-j/pytorch-cpu/INT4
+pip install -r requirements.txt
+bash run_calibration_int4.sh
+
+test $? -eq 0 || exit $?
diff --git a/cm-mlops/script/get-ml-model-gptj/run-intel.sh b/cm-mlops/script/get-ml-model-gptj/run-intel.sh
new file mode 100644
index 0000000000..f6cb2134da
--- /dev/null
+++ b/cm-mlops/script/get-ml-model-gptj/run-intel.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+export PATH=${CM_CONDA_BIN_PATH}:$PATH
+
+export CALIBRATION_DATA_JSON=${CM_CALIBRATION_DATASET_CNNDM_PATH}
+
+
+if [[ ${CM_ML_MODEL_WEIGHT_DATA_TYPES} == "int4" ]]; then
+ export INT4_CALIBRATION_DIR=${PWD}/quantized-int4-model
+ bash ${CM_TMP_CURRENT_SCRIPT_PATH}/run-int4-calibration.sh
+ cd ${CM_HARNESS_CODE_ROOT}
+ bash run_quantization_int4.sh
+else
+ cd ${CM_HARNESS_CODE_ROOT}
+ bash run_quantization.sh
+fi
+
+test $? -eq 0 || exit $?
diff --git a/cm-mlops/script/get-python3/_cm.json b/cm-mlops/script/get-python3/_cm.json
index 5ab8b7defb..85066e694d 100644
--- a/cm-mlops/script/get-python3/_cm.json
+++ b/cm-mlops/script/get-python3/_cm.json
@@ -41,6 +41,30 @@
],
"uid": "d0b5dd74373f4a62",
"variations": {
+ "custom-path.#": {
+ "env": {
+ "CM_PYTHON_BIN_WITH_PATH": "#"
+ }
+ },
+ "conda.#": {
+ "deps": [
+ {
+ "tags": "get,generic,conda-package,_name.#,_package.python",
+ "names": [
+ "conda-package",
+ "conda-python"
+ ]
+ }
+ ],
+ "adr": {
+ "pip-package": {
+ "tags": "_conda.#"
+ }
+ },
+ "env": {
+ "CM_PYTHON_CONDA": "yes"
+ }
+ },
"lto": {},
"optimized": {},
"shared": {},
diff --git a/cm-mlops/script/get-python3/customize.py b/cm-mlops/script/get-python3/customize.py
index 11373dba80..3e2cc0b6f4 100644
--- a/cm-mlops/script/get-python3/customize.py
+++ b/cm-mlops/script/get-python3/customize.py
@@ -7,6 +7,9 @@ def preprocess(i):
env = i['env']
+ if env.get('CM_PYTHON_CONDA', '') == 'yes':
+ env['CM_PYTHON_BIN_WITH_PATH'] = os.path.join(env['CM_CONDA_BIN_PATH'], "python")
+
recursion_spaces = i['recursion_spaces']
# we need to understand whether this script is called first and CM_PYTHON_BIN_WITH_PATH is empty
diff --git a/cm-mlops/script/gui/app.py b/cm-mlops/script/gui/app.py
index 49546be600..70e81b548f 100644
--- a/cm-mlops/script/gui/app.py
+++ b/cm-mlops/script/gui/app.py
@@ -217,6 +217,7 @@ def main():
tags = '--tags={}'.format(x)
# Check flags
+ flags_dict = {}
flags = ''
for key in st_inputs:
value = st_inputs[key]
@@ -224,19 +225,30 @@ def main():
if value!='' and (type(value)!=bool or value==True):
flags+=' '+var1+'\n --'+key2
+
+ z = True
if type(value)!=bool:
x = str(value)
+ z = x
+
if ' ' in x or ':' in x or '/' in x or '\\' in x:
x='"'+x+'"'
flags+='='+x
+ flags_dict[key2]=z
+
########################################################
# Extra CMD
st.markdown("""---""")
cmd_extension = st.text_input("Add extra to CM script command line").strip()
+ run_via_docker = False
+ if len(meta.get('docker',{}))>0:
+ run_via_docker = st.checkbox('Run via Docker', key='run_via_docker', value=False)
+
# Prepare CLI
- cli = 'cm run script {} {} '.format(tags, flags,)
+ action = 'docker' if run_via_docker else 'run'
+ cli = 'cm {} script {} {} '.format(action, tags, flags,)
x = '' if cmd_extension=='' else var1+'\n '+cmd_extension
@@ -248,6 +260,7 @@ def main():
# if no_run=='':
# cli+=' --pause\n'
+
# Print CLI
st.markdown("""---""")
@@ -278,7 +291,44 @@ def main():
st.text_area('**Install [CM interface](https://github.com/mlcommons/ck) with a few dependencies:**', x, height=170)
- cli = st.text_area('**Run CM script:**', cli, height=500)
+
+ st.markdown("**Run CM script from Python:**")
+
+ # Python API
+ dd = {
+ 'action':action,
+ 'automation':'script,5b4e0237da074764',
+ 'tags':script_tags
+ }
+
+ x = script_tags
+ if len(selected_variations)>0:
+ for sv in selected_variations:
+ x+=' '+sv
+
+ dd['tags']=x.replace(' ',',')
+
+ dd.update(flags_dict)
+
+ import json
+ dd_json=json.dumps(dd, indent=2)
+
+ y = 'import cmind\n'
+ y+= 'r = cmind.access('+dd_json+')\n'
+ y+= 'if r[\'return\']>0: print (r)\n'
+
+ x='''
+ ```python
+ {}
+ '''.format(y)
+
+# st.write(x.replace('\n','
\n'), unsafe_allow_html=True)
+
+ st.markdown(x)
+
+
+ cli = st.text_area('**Run CM script from command line:**', cli, height=500)
+
# Add explicit button "Run"
diff --git a/cm-mlops/script/install-llvm-prebuilt/README-extra.md b/cm-mlops/script/install-llvm-prebuilt/README-extra.md
index 34c4385c4d..1ad1e122be 100644
--- a/cm-mlops/script/install-llvm-prebuilt/README-extra.md
+++ b/cm-mlops/script/install-llvm-prebuilt/README-extra.md
@@ -13,6 +13,7 @@ This [CM script](https://github.com/mlcommons/ck/blob/master/cm/docs/specs/scrip
## Supported and Tested OS
1. Ubuntu 18.04, 20.04, 22.04
2. RHEL 9
+3. Windows 10, 11
# CLI
@@ -58,6 +59,19 @@ cm run script "app image corner-detection"
* Windows, Linux, MacOS
+### Ubuntu 22.04
+
+* 17.0.6
+* 17.0.5
+* 17.0.4
+* 17.0.2
+* 16.0.4
+* 16.0.0
+ `sudo apt install libncurses5`
+* 15.0.6
+* 14.0.0
+
+
### RHEL 9
#### v14.0.0: ✓
diff --git a/cm-mlops/script/install-llvm-prebuilt/_cm.json b/cm-mlops/script/install-llvm-prebuilt/_cm.json
index 4db0625ab4..39cef74498 100644
--- a/cm-mlops/script/install-llvm-prebuilt/_cm.json
+++ b/cm-mlops/script/install-llvm-prebuilt/_cm.json
@@ -4,7 +4,7 @@
"category": "Compiler automation",
"automation_uid": "5b4e0237da074764",
"cache": true,
- "default_version": "14.0.0",
+ "default_version": "15.0.6",
"deps": [
{
"tags": "detect,os"
diff --git a/cm-mlops/script/install-llvm-prebuilt/customize.py b/cm-mlops/script/install-llvm-prebuilt/customize.py
index f2ac97f67b..25cd6b922f 100644
--- a/cm-mlops/script/install-llvm-prebuilt/customize.py
+++ b/cm-mlops/script/install-llvm-prebuilt/customize.py
@@ -61,7 +61,22 @@ def preprocess(i):
# Treat all Linux flavours as Ubuntu for now ...
if True:
- default_os='18.04'
+ need_version_split = need_version.split('.')
+
+ default_os = '22.04'
+
+ if len(need_version_split)>0:
+ hver = 0
+ try:
+ hver = int(need_version_split[0])
+ except:
+ pass
+
+ if hver>0:
+ if hver<16:
+ default_os='18.04'
+ else:
+ default_os='22.04'
if need_version == '10.0.1':
default_os = '16.04'
@@ -95,7 +110,17 @@ def preprocess(i):
elif need_version == '14.0.0':
default_os = '18.04'
- package_name = 'clang+llvm-' + need_version + '-x86_64-linux-gnu-ubuntu-' + default_os + '.tar.xz'
+ elif need_version == '15.0.6':
+ default_os = '18.04'
+
+ elif need_version == '16.0.0':
+ default_os = '18.04'
+
+ elif need_version == '16.0.4':
+ default_os = '22.04'
+
+
+ package_name = 'clang+llvm-' + need_version + '-x86_64-linux-gnu-ubuntu-' + default_os + '.tar.xz'
package_url = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-' + need_version + '/' + package_name
diff --git a/cm-mlops/script/print-hello-world-py/_cm.json b/cm-mlops/script/print-hello-world-py/_cm.json
index 818760ded1..5e6a0629c7 100644
--- a/cm-mlops/script/print-hello-world-py/_cm.json
+++ b/cm-mlops/script/print-hello-world-py/_cm.json
@@ -18,7 +18,11 @@
"tags": "get,python3"
},
{
- "tags": "print,python-version"
+ "tags": "print,python-version",
+ "skip_if_env": {
+ "CM_SKIP_PRINT":["True"],
+ "CM_SKIP_PRINT2":["True"]
+ }
}
],
"tags": [
diff --git a/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml
index 49607eff8f..5011cec210 100644
--- a/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml
+++ b/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml
@@ -138,6 +138,8 @@ deps:
# Install MLPerf inference dependencies
- tags: get,mlperf,inference,results
+ names:
+ inference-results
version: v3.1
@@ -253,13 +255,15 @@ variations:
CM_BENCHMARK: STANDALONE_GPTJ
gptj_,pytorch:
+ adr:
+ conda-package:
+ tags: _name.gptj-pt
deps:
- tags: get,conda,_name.gptj-pt
- - tags: get,generic,conda-package,_package.python
- names:
- - conda-package
- - python
- version: "3.9"
+ - tags: get,python,_conda.gptj-pt
+ adr:
+ conda-python:
+ version: "3.9"
- tags: install,llvm,src,_tag.llvmorg-16.0.6,_clang,_release,_for-intel-mlperf-inference-v3.1-gptj
- names:
- conda-package
@@ -271,9 +275,9 @@ variations:
- conda-package
- jemalloc
- tags: install,ipex,from.src,_for-intel-mlperf-inference-v3.1-gptj
- - tags: get,generic-python-lib,_custom-python,_package.transformers
- env:
- CM_PYTHON_BIN_WITH_PATH: "<<>>/python3"
+ - tags: get,generic-python-lib,_package.transformers
+ names:
+ - pip-package
version: "4.28.1"
- tags: get,mlcommons,inference,src
names:
@@ -288,12 +292,12 @@ variations:
- ml-model
- gptj-model
- gpt-j-model
- - tags: get,generic-python-lib,_custom-python,_package.datasets
- env:
- CM_PYTHON_BIN_WITH_PATH: "<<>>/python3"
- - tags: get,generic-python-lib,_custom-python,_package.accelerate
- env:
- CM_PYTHON_BIN_WITH_PATH: "<<>>/python3"
+ - tags: get,generic-python-lib,_package.datasets
+ names:
+ - pip-package
+ - tags: get,generic-python-lib,_package.accelerate
+ names:
+ - pip-package
gptj-99:
group: model
diff --git a/cm-mlops/script/run-mlperf-inference-app/customize.py b/cm-mlops/script/run-mlperf-inference-app/customize.py
index eb96904b14..56a6624c44 100644
--- a/cm-mlops/script/run-mlperf-inference-app/customize.py
+++ b/cm-mlops/script/run-mlperf-inference-app/customize.py
@@ -98,7 +98,8 @@ def preprocess(i):
test_list = ["TEST01", "TEST05"]
if env['CM_MODEL'] in ["resnet50"]:
test_list.append("TEST04")
- if "gpt" in env['CM_MODEL']:
+ if "gpt" in env['CM_MODEL'] or "sdxl" in env['CM_MODEL'] or "llama2-70b" in env['CM_MODEL']:
+ test_list.remove("TEST01")
test_list.remove("TEST05")
variation_implementation= "_" + env.get("CM_MLPERF_IMPLEMENTATION", "reference")
diff --git a/cm/README.md b/cm/README.md
index 99e6e2ccf3..70d9c1c781 100644
--- a/cm/README.md
+++ b/cm/README.md
@@ -17,7 +17,7 @@ either natively or inside containers.
Here are a few most commonly used examples from the CM users
that should run in the same way on Linux, MacOS, Windows and other platforms
-(see [Gettings Started Guide](docs/getting-started.md) to understand
+(see [Getting Started Guide](docs/getting-started.md) to understand
how they work and how to reuse them in your projects):
```bash
@@ -30,6 +30,7 @@ cmr "python app image-classification onnx"
cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e
cmr "python app image-classification onnx" --input=computer_mouse.jpg
+cmr "python app image-classification onnx" --input=computer_mouse.jpg --debug
cm find script "python app image-classification onnx"
@@ -38,7 +39,8 @@ cmr "install python-venv" --version_max=3.10.11 --name=mlperf
cmr "get ml-model stable-diffusion"
cmr "get ml-model huggingface zoo _model-stub.alpindale/Llama-2-13b-ONNX" --model_filename=FP32/LlamaV2_13B_float32.onnx --skip_cache
-cmr "get coco dataset _val _2014"
+cmr "get dataset coco _val _2014"
+cmr "get dataset openimages" -j
cm show cache
cm show cache "get ml-model stable-diffusion"
@@ -49,17 +51,29 @@ cmr "python app image-classification onnx" --input=computer_mouse.jpg
cm rm cache -f
cmr "python app image-classification onnx" --input=computer_mouse.jpg --adr.onnxruntime.version_max=1.16.0
-
+cmr "get cuda" --version_min=12.0.0 --version_max=12.3.1
cmr "python app image-classification onnx _cuda" --input=computer_mouse.jpg
-cmr "cm gui" --script="python app image-classification onnx"
+cm gui script "python app image-classification onnx"
cm docker script "python app image-classification onnx" --input=computer_mouse.jpg
cm docker script "python app image-classification onnx" --input=computer_mouse.jpg -j -docker_it
+cm docker script "get coco dataset _val _2017" --to=d:\Downloads\COCO-2017-val --store=d:\Downloads --docker_cm_repo=ctuning@mlcommons-ck
+
cmr "run common mlperf inference" --implementation=nvidia --model=bert-99 --category=datacenter --division=closed
cm find script "run common mlperf inference"
+cmr "get generic-python-lib _package.torch" --version=2.1.1
+cmr "get generic-python-lib _package.torchvision" --version=0.16.2
+cmr "python app image-classification torch" --input=computer_mouse.jpg
+
+
+cmr "install llvm prebuilt" --version=17.0.6
+cmr "app image corner-detection"
+
+cmr "get conda"
+
cm pull repo ctuning@cm-reproduce-research-projects
cmr "reproduce paper micro-2023 victima _install_deps"
cmr "reproduce paper micro-2023 victima _run"
diff --git a/platform/README.md b/platform/README.md
index 6f8499e1f4..e75de5c89a 100644
--- a/platform/README.md
+++ b/platform/README.md
@@ -1,6 +1,7 @@
# Collective Knowledge Playground
-*Note that this project is under heavy development.*
+*Note that this project is under heavy development.
+ We are preparing a major update based on very useful feedback from our users during MLPerf inference 3.1 community submission!*
### Introduction