diff --git a/README.md b/README.md index 7cbf3b65e2..d447da2cfd 100755 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ either natively or inside containers. Here are a few most commonly used examples from the CM users that should run in the same way on Linux, MacOS, Windows and other platforms -(see [Gettings Started Guide](docs/getting-started.md) to understand +(see [Getting Started Guide](docs/getting-started.md) to understand how they work and how to reuse them in your projects): ```bash @@ -38,6 +38,7 @@ cmr "python app image-classification onnx" cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e cmr "python app image-classification onnx" --input=computer_mouse.jpg +cmr "python app image-classification onnx" --input=computer_mouse.jpg --debug cm find script "python app image-classification onnx" @@ -46,7 +47,8 @@ cmr "install python-venv" --version_max=3.10.11 --name=mlperf cmr "get ml-model stable-diffusion" cmr "get ml-model huggingface zoo _model-stub.alpindale/Llama-2-13b-ONNX" --model_filename=FP32/LlamaV2_13B_float32.onnx --skip_cache -cmr "get coco dataset _val _2014" +cmr "get dataset coco _val _2014" +cmr "get dataset openimages" -j cm show cache cm show cache "get ml-model stable-diffusion" @@ -57,21 +59,34 @@ cmr "python app image-classification onnx" --input=computer_mouse.jpg cm rm cache -f cmr "python app image-classification onnx" --input=computer_mouse.jpg --adr.onnxruntime.version_max=1.16.0 - +cmr "get cuda" --version_min=12.0.0 --version_max=12.3.1 cmr "python app image-classification onnx _cuda" --input=computer_mouse.jpg -cmr "cm gui" --script="python app image-classification onnx" +cm gui script "python app image-classification onnx" cm docker script "python app image-classification onnx" --input=computer_mouse.jpg cm docker script "python app image-classification onnx" --input=computer_mouse.jpg -j -docker_it +cm docker script "get coco dataset _val _2017" --to=d:\Downloads\COCO-2017-val --store=d:\Downloads --docker_cm_repo=ctuning@mlcommons-ck + cmr "run common mlperf inference" --implementation=nvidia --model=bert-99 --category=datacenter --division=closed cm find script "run common mlperf inference" +cmr "get generic-python-lib _package.torch" --version=2.1.1 +cmr "get generic-python-lib _package.torchvision" --version=0.16.2 +cmr "python app image-classification torch" --input=computer_mouse.jpg + + +cmr "install llvm prebuilt" --version=17.0.6 +cmr "app image corner-detection" + +cmr "get conda" + cm pull repo ctuning@cm-reproduce-research-projects cmr "reproduce paper micro-2023 victima _install_deps" cmr "reproduce paper micro-2023 victima _run" + ``` ```python diff --git a/cm-mlops/automation/script/module.py b/cm-mlops/automation/script/module.py index 4f81ece1ad..6a2ce81899 100644 --- a/cm-mlops/automation/script/module.py +++ b/cm-mlops/automation/script/module.py @@ -127,6 +127,7 @@ def run(self, i): (save_env) (bool): if True, save env and state to tmp-env.sh/bat and tmp-state.json (shell) (bool): if True, save env with cmd/bash and run it + (debug) (bool): the same as shell (recursion) (bool): True if recursive call. Useful when preparing the global bat file or Docker container @@ -180,7 +181,7 @@ def run(self, i): * state (dict): global state (updated by this script - includes new_state) """ - + from cmind import utils import copy import time @@ -279,9 +280,14 @@ def run(self, i): skip_cache = i.get('skip_cache', False) force_cache = i.get('force_cache', False) + fake_run = i.get('fake_run', False) fake_run = i.get('fake_run', False) if 'fake_run' in i else i.get('prepare', False) + if fake_run: env['CM_TMP_FAKE_RUN']='yes' + fake_deps = i.get('fake_deps', False) + if fake_deps: env['CM_TMP_FAKE_DEPS']='yes' + run_state = i.get('run_state', self.run_state) if run_state.get('version_info', '') == '': run_state['version_info'] = {} @@ -289,6 +295,7 @@ def run(self, i): run_state['parent'] = None if fake_deps: run_state['fake_deps'] = True + print_deps = i.get('print_deps', False) print_readme = i.get('print_readme', False) @@ -454,9 +461,10 @@ def run(self, i): cm_script_info += '"' - if verbose: - print ('') +# if verbose: +# print ('') + print ('') print (recursion_spaces + '* ' + cm_script_info) @@ -1097,6 +1105,8 @@ def run(self, i): if r['return']>0: return r version = r['meta'].get('version') + print (recursion_spaces + ' ! load {}'.format(path_to_cached_state_file)) + ################################################################################################ # IF REUSE FROM CACHE - update env and state from cache! @@ -1521,7 +1531,17 @@ def run(self, i): utils.merge_dicts({'dict1':env, 'dict2':const, 'append_lists':True, 'append_unique':True}) utils.merge_dicts({'dict1':state, 'dict2':const_state, 'append_lists':True, 'append_unique':True}) - r = detect_state_diff(env, saved_env, new_env_keys_from_meta, new_state_keys_from_meta, state, saved_state) + if i.get('force_new_env_keys', []): + new_env_keys = i['force_new_env_keys'] + else: + new_env_keys = new_env_keys_from_meta + + if i.get('force_new_state_keys', []): + new_state_keys = i['force_new_state_keys'] + else: + new_state_keys = new_state_keys_from_meta + + r = detect_state_diff(env, saved_env, new_env_keys, new_state_keys, state, saved_state) if r['return']>0: return r new_env = r['new_env'] @@ -1609,11 +1629,14 @@ def run(self, i): script_path = os.getcwd() os.chdir(current_path) - if not i.get('dirty', False) and not cache: + shell = i.get('shell', False) + if not shell: + shell = i.get('debug', False) + + if not shell and not i.get('dirty', False) and not cache: clean_tmp_files(clean_files, recursion_spaces) # Record new env and new state in the current dir if needed - shell = i.get('shell', False) if save_env or shell: # Check if script_prefix in the state from other components where_to_add = len(os_info['start_script']) @@ -1625,13 +1648,15 @@ def run(self, i): env_script.insert(where_to_add, x) if shell: - x = 'cmd' if os_info['platform'] == 'windows' else 'bash' + x=['cmd', '.', '','.bat',''] if os_info['platform'] == 'windows' else ['bash', ' ""', '"','.sh','. ./'] env_script.append('\n') - env_script.append('echo "Working path: {}"'.format(script_path)) - env_script.append('echo "Running debug shell. Type exit to quit ..."\n') + env_script.append('echo{}\n'.format(x[1])) + env_script.append('echo {}Working path: {}{}'.format(x[2], script_path, x[2])) + env_script.append('echo {}Running debug shell. Change and run "tmp-run{}". Type exit to quit ...{}\n'.format(x[2],x[3],x[2])) + env_script.append('echo{}\n'.format(x[1])) env_script.append('\n') - env_script.append(x) + env_script.append(x[0]) env_file = self.tmp_file_env + bat_ext @@ -2154,7 +2179,7 @@ def add(self, i): console = i.get('out') == 'con' - # Try to find experiment artifact by alias and/or tags + # Try to find script artifact by alias and/or tags ii = utils.sub_input(i, self.cmind.cfg['artifact_keys']) parsed_artifact = i.get('parsed_artifact',[]) @@ -2511,6 +2536,9 @@ def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, a if not d.get('tags'): continue + if d.get('skip_if_fake_run', False) and env.get('CM_TMP_FAKE_RUN','')=='yes': + continue + if "enable_if_env" in d: if not enable_or_skip_script(d["enable_if_env"], env): continue @@ -3408,7 +3436,7 @@ def get_default_path_list(self, i): ############################################################ def doc(self, i): """ - Add CM automation. + Document CM script. Args: (CM input dict): @@ -3433,11 +3461,47 @@ def doc(self, i): return utils.call_internal_module(self, __file__, 'module_misc', 'doc', i) + ############################################################ + def gui(self, i): + """ + Run GUI for CM script. + + Args: + (CM input dict): + + Returns: + (CM return dict): + + * return (int): return code == 0 if no error and >0 if error + * (error) (str): error string if return>0 + + """ + + artifact = i.get('artifact', '') + tags = '' + if artifact != '': + if ' ' in artifact: + tags = artifact.replace(' ',',') + + if tags=='': + tags = i.get('tags','') + + if 'tags' in i: + del(i['tags']) + + i['action']='run' + i['artifact']='gui' + i['parsed_artifact']=[('gui','605cac42514a4c69')] + i['script']=tags.replace(',',' ') + + return self.cmind.access(i) + + ############################################################ def dockerfile(self, i): """ - Add CM automation. + Generate Dockerfile for CM script. Args: (CM input dict): @@ -3465,7 +3529,7 @@ def dockerfile(self, i): ############################################################ def docker(self, i): """ - Add CM automation. + Run CM script in an automatically-generated container. Args: (CM input dict): @@ -3935,6 +3999,10 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): print (recursion_spaces + ' - Running native script "{}" from temporal script "{}" in "{}" ...'.format(path_to_run_script, run_script, cur_dir)) print ('') + print (recursion_spaces + ' ! cd {}'.format(cur_dir)) + print (recursion_spaces + ' ! call {} from {}'.format(run_script, path_to_run_script)) + + # Prepare env variables import copy script = copy.deepcopy(os_info['start_script']) @@ -3949,12 +4017,14 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): # Check if run bash/cmd before running the command (for debugging) if debug_script_tags !='' and all(item in found_script_tags for item in debug_script_tags.split(',')): - x = 'cmd' if os_info['platform'] == 'windows' else 'bash' + x=['cmd', '.', '','.bat'] if os_info['platform'] == 'windows' else ['bash', ' ""', '"','.sh'] script.append('\n') - script.append('echo "Running debug shell. Type exit to quit ..."\n') + script.append('echo{}\n'.format(x[1])) + script.append('echo {}Running debug shell. Type exit to resume script execution ...{}\n'.format(x[2],x[3],x[2])) + script.append('echo{}\n'.format(x[1])) script.append('\n') - script.append(x) + script.append(x[0]) # Append batch file to the tmp script script.append('\n') @@ -4013,6 +4083,11 @@ def prepare_and_run_script_with_postprocessing(i, postprocess="postprocess"): utils.merge_dicts({'dict1':env, 'dict2':updated_env, 'append_lists':True, 'append_unique':True}) + + if customize_code is not None: + print (recursion_spaces+' ! call "{}" from {}'.format(postprocess, customize_code.__file__)) + + if len(posthook_deps)>0 and (postprocess == "postprocess"): r = script_automation._call_run_deps(posthook_deps, local_env_keys, local_env_keys_from_meta, env, state, const, const_state, add_deps_recursive, recursion_spaces, remembered_selections, variation_tags_string, found_cached, debug_script_tags, verbose, run_state) diff --git a/cm-mlops/automation/script/module_misc.py b/cm-mlops/automation/script/module_misc.py index a0e55b8408..92da5d42d1 100644 --- a/cm-mlops/automation/script/module_misc.py +++ b/cm-mlops/automation/script/module_misc.py @@ -1051,6 +1051,47 @@ def doc(i): +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +def update_path_for_docker(path, mounts, force_path_target=''): + + path_orig = '' + path_target = '' + + if path!='': # and (os.path.isfile(path) or os.path.isdir(path)): + path = os.path.abspath(path) + + path_target = path + path_orig = path + + if os.name == 'nt': + from pathlib import PureWindowsPath, PurePosixPath + + x = PureWindowsPath(path_orig) + path_target = str(PurePosixPath('/', *x.parts[1:])) + + if not path_target.startswith('/'): path_target='/'+path_target + + path_target='/cm-mount'+path_target if force_path_target=='' else force_path_target + + # If file, mount directory + if os.path.isfile(path): + x = os.path.dirname(path_orig) + ':' + os.path.dirname(path_target) + else: + x = path_orig + ':' + path_target + + # CHeck if no duplicates + to_add = True + for y in mounts: + if y.lower()==x.lower(): + to_add = False + break + + if to_add: + mounts.append(x) + + + return (path_orig, path_target) + ############################################################ def process_inputs(i): @@ -1063,34 +1104,6 @@ def process_inputs(i): # Check if need to update/map/mount inputs and env i_run_cmd = copy.deepcopy(i_run_cmd_arc) - # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - def update_path_for_docker(path, mounts): - - path_orig = '' - path_target = '' - - if path!='' and (os.path.isfile(path) or os.path.isdir(path)): - path = os.path.abspath(path) - - path_target = path - path_orig = path - - if os.name == 'nt': - from pathlib import PureWindowsPath, PurePosixPath - - x = PureWindowsPath(path_orig) - path_target = str(PurePosixPath('/', *x.parts[1:])) - - if not path_target.startswith('/'): path_target='/'+path_target - path_target='/cm-mount'+path_target - - # If file, mount directory - if os.path.isfile(path): - mounts.append(os.path.dirname(path_orig) + ':' + os.path.dirname(path_target)) - else: - mounts.append(path_orig + ':' + path_target) - - return (path_orig, path_target) def get_value_using_key_with_dots(d, k): v = None @@ -1570,6 +1583,11 @@ def docker(i): i_run_cmd = r['run_cmd'] + # Check if need to mount home directory + current_path_target = '/cm-mount/current' + if docker_settings.get('mount_current_dir','')=='yes': + update_path_for_docker('.', mounts, force_path_target=current_path_target) + _os = i.get('docker_os', meta.get('docker_os', 'ubuntu')) version = i.get('docker_os_version', meta.get('docker_os_version', '22.04')) @@ -1680,7 +1698,10 @@ def docker(i): if r['return']>0: return r run_cmd = r['run_cmd_string'] - + + if docker_settings.get('mount_current_dir','')=='yes': + run_cmd = 'cd '+current_path_target+' && '+run_cmd + print ('') print ('CM command line regenerated to be used inside Docker:') print ('') diff --git a/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml b/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml index c3f869081e..b3fbb7bf93 100644 --- a/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml +++ b/cm-mlops/script/app-image-classification-onnx-py/_cm.yaml @@ -65,9 +65,15 @@ deps: variations: cuda: + group: target env: USE_CUDA: yes + cpu: + group: target + default: yes + env: + USE_CPU: yes input_mapping: input: CM_IMAGE diff --git a/cm-mlops/script/app-image-classification-onnx-py/customize.py b/cm-mlops/script/app-image-classification-onnx-py/customize.py index e1c07dc2e3..43098c71f7 100644 --- a/cm-mlops/script/app-image-classification-onnx-py/customize.py +++ b/cm-mlops/script/app-image-classification-onnx-py/customize.py @@ -53,6 +53,12 @@ def postprocess(i): except Exception as e: print ('CM warning: {}'.format(e)) + top_classification = data.get('top_classification','') + + if top_classification!='': + print ('') + x = 'Top classification: {}'.format(top_classification) + print ('='*len(x)) + print (x) - return {'return':0} diff --git a/cm-mlops/script/app-image-classification-torch-py/README-extra.md b/cm-mlops/script/app-image-classification-torch-py/README-extra.md index b9eb292733..6628885061 100644 --- a/cm-mlops/script/app-image-classification-torch-py/README-extra.md +++ b/cm-mlops/script/app-image-classification-torch-py/README-extra.md @@ -1,12 +1,16 @@ -# CUDA version +# CPU + +## 20240129; Windows 11 ```bash -cm run script "install python-venv" --name=test -cm run script "python app image-classification pytorch _cuda" -cm run script "python app image-classification pytorch _cuda" --input=src/computer_mouse.jpg +cmr "get generic-python-lib _package.torch" --version=2.1.1 +cmr "get generic-python-lib _package.torchvision" --version=0.16.2 ``` -## Tested on Windows 10 -* ONNX Runtime 1.13.1 with CUDA -* CUDA 11.6 -* cuDNN 8.5.0.96 +# CUDA + +```bash +cm run script "install python-venv" --name=test +cm run script "python app image-classification pytorch _cuda" --adr.python.name=test +cm run script "python app image-classification pytorch _cuda" --adr.python.name=test --input=src/computer_mouse.jpg +``` diff --git a/cm-mlops/script/app-image-corner-detection/README-extra.md b/cm-mlops/script/app-image-corner-detection/README-extra.md index 8bcff69f95..6161c6521c 100644 --- a/cm-mlops/script/app-image-corner-detection/README-extra.md +++ b/cm-mlops/script/app-image-corner-detection/README-extra.md @@ -6,3 +6,9 @@ cm run script "app image corner-detection" -add_deps_recursive.compiler.tags=llv cm run script "app image corner-detection" -add_deps_recursive.compiler.tags=gcc cm run script "app image corner-detection" -add_deps_recursive.compiler.tags=llvm --add_deps_recursive.compiler.version_min=11.0.0 --add_deps_recursive.compiler.version_max=13.0.0 ``` + +## Reproducibility matrix + +* Ubuntu 22.04; x64; LLVM 17.06 +* Windows 11; x64; LLVM 17.06 + diff --git a/cm-mlops/script/app-mlperf-inference-reference/customize.py b/cm-mlops/script/app-mlperf-inference-reference/customize.py index df880c219d..c22344df03 100644 --- a/cm-mlops/script/app-mlperf-inference-reference/customize.py +++ b/cm-mlops/script/app-mlperf-inference-reference/customize.py @@ -66,10 +66,11 @@ def preprocess(i): env['CM_MLPERF_CONF'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "mlperf.conf") + x="" if os_info['platform'] == 'windows' else "'" if "llama2-70b" in env['CM_MODEL']: - env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf-conf '" + env['CM_MLPERF_CONF'] + "'" + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf-conf " + x+ env['CM_MLPERF_CONF'] + x else: - env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf_conf '" + env['CM_MLPERF_CONF'] + "'" + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] += " --mlperf_conf "+ x + env['CM_MLPERF_CONF'] + x env['MODEL_DIR'] = env.get('CM_ML_MODEL_PATH') if not env['MODEL_DIR']: @@ -95,10 +96,11 @@ def preprocess(i): ml_model_name = env['CM_MODEL'] if 'CM_MLPERF_USER_CONF' in env: user_conf_path = env['CM_MLPERF_USER_CONF'] + x="" if os_info['platform'] == 'windows' else "'" if 'llama2-70b' in env['CM_MODEL']: - scenario_extra_options += " --user-conf '" + user_conf_path + "'" + scenario_extra_options += " --user-conf " + x + user_conf_path + x else: - scenario_extra_options += " --user_conf '" + user_conf_path + "'" + scenario_extra_options += " --user_conf " + x + user_conf_path + x mode = env['CM_MLPERF_LOADGEN_MODE'] mode_extra_options = "" @@ -139,7 +141,7 @@ def preprocess(i): env['CM_MLPERF_OUTPUT_DIR'] = os.getcwd() mlperf_implementation = env.get('CM_MLPERF_IMPLEMENTATION', 'reference') - cmd, run_dir = get_run_cmd(env, scenario_extra_options, mode_extra_options, dataset_options, mlperf_implementation) + cmd, run_dir = get_run_cmd(os_info, env, scenario_extra_options, mode_extra_options, dataset_options, mlperf_implementation) if env.get('CM_NETWORK_LOADGEN', '') == "lon": @@ -158,15 +160,15 @@ def preprocess(i): return {'return':0} -def get_run_cmd(env, scenario_extra_options, mode_extra_options, dataset_options, implementation="reference"): +def get_run_cmd(os_info, env, scenario_extra_options, mode_extra_options, dataset_options, implementation="reference"): if implementation == "reference": - return get_run_cmd_reference(env, scenario_extra_options, mode_extra_options, dataset_options) + return get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_options, dataset_options) if implementation == "nvidia": - return get_run_cmd_nvidia(env, scenario_extra_options, mode_extra_options, dataset_options) + return get_run_cmd_nvidia(os_info, env, scenario_extra_options, mode_extra_options, dataset_options) return "", os.getcwd() -def get_run_cmd_reference(env, scenario_extra_options, mode_extra_options, dataset_options): +def get_run_cmd_reference(os_info, env, scenario_extra_options, mode_extra_options, dataset_options): if env['CM_MODEL'] in [ "gptj-99", "gptj-99.9" ]: @@ -190,9 +192,15 @@ def get_run_cmd_reference(env, scenario_extra_options, mode_extra_options, datas env['RUN_DIR'] = os.path.join(env['CM_MLPERF_INFERENCE_SOURCE'], "vision", "classification_and_detection") env['OUTPUT_DIR'] = env['CM_MLPERF_OUTPUT_DIR'] if env.get('CM_MLPERF_VISION_DATASET_OPTION','') == '' and env.get('CM_MLPERF_DEVICE') != "tpu": - cmd = "./run_local.sh " + env['CM_MLPERF_BACKEND'] + ' ' + \ - env['CM_MODEL'] + ' ' + env['CM_MLPERF_DEVICE'] + " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + " " + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ - scenario_extra_options + mode_extra_options + dataset_options + if os_info['platform'] == 'windows': + cmd = "python python/main.py --profile "+env['CM_MODEL']+"-"+env['CM_MLPERF_BACKEND'] + \ + " --model=" + env['CM_ML_MODEL_FILE_WITH_PATH'] + ' --dataset-path=' + env['CM_DATASET_PREPROCESSED_PATH'] + \ + " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + " " + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ + scenario_extra_options + mode_extra_options + dataset_options + else: + cmd = "./run_local.sh " + env['CM_MLPERF_BACKEND'] + ' ' + \ + env['CM_MODEL'] + ' ' + env['CM_MLPERF_DEVICE'] + " --scenario " + env['CM_MLPERF_LOADGEN_SCENARIO'] + " " + env['CM_MLPERF_LOADGEN_EXTRA_OPTIONS'] + \ + scenario_extra_options + mode_extra_options + dataset_options return cmd, env['RUN_DIR'] if env['CM_MLPERF_BACKEND'] == "ncnn": diff --git a/cm-mlops/script/benchmark-program-mlperf/customize.py b/cm-mlops/script/benchmark-program-mlperf/customize.py index e08823fe5a..a333b6c078 100644 --- a/cm-mlops/script/benchmark-program-mlperf/customize.py +++ b/cm-mlops/script/benchmark-program-mlperf/customize.py @@ -9,9 +9,16 @@ def preprocess(i): def postprocess(i): + os_info = i['os_info'] env = i['env'] + env['CM_MLPERF_RUN_CMD'] = env.get('CM_RUN_CMD') + + if env.get('CM_MLPERF_POWER', '') == "yes": + if os_info['platform'] == 'windows': + return {'return':1, 'error':'TBD: this script is not yet supported on Windows'} + if env.get('CM_MLPERF_SHORT_RANGING_RUN', '') != 'no': os.system("echo '0' > "+env.get('CM_RUN_DIR','')+ "/count.txt") env['CM_MLPERF_RUN_CMD'] = "CM_MLPERF_RUN_COUNT=\$(cat \${CM_RUN_DIR}/count.txt); echo \${CM_MLPERF_RUN_COUNT}; CM_MLPERF_RUN_COUNT=\$((CM_MLPERF_RUN_COUNT+1)); echo \${CM_MLPERF_RUN_COUNT} > \${CM_RUN_DIR}/count.txt && if [ \${CM_MLPERF_RUN_COUNT} -eq \'1\' ]; then export CM_MLPERF_USER_CONF=\${CM_MLPERF_RANGING_USER_CONF}; else export CM_MLPERF_USER_CONF=\${CM_MLPERF_TESTING_USER_CONF}; fi && "+env.get('CM_RUN_CMD','').strip() diff --git a/cm-mlops/script/benchmark-program/run.bat b/cm-mlops/script/benchmark-program/run.bat index e00f449760..62929b0e8e 100644 --- a/cm-mlops/script/benchmark-program/run.bat +++ b/cm-mlops/script/benchmark-program/run.bat @@ -2,9 +2,6 @@ cd %CM_RUN_DIR% -echo %CM_RUN_CMD% -pause - %CM_RUN_CMD% IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/cm-mlops/script/get-conda/_cm.json b/cm-mlops/script/get-conda/_cm.json index eef50fa6cf..3253a3ff56 100644 --- a/cm-mlops/script/get-conda/_cm.json +++ b/cm-mlops/script/get-conda/_cm.json @@ -29,11 +29,15 @@ "name.#": { "env": { "CM_CONDA_PREFIX_NAME": "#" + }, + "adr": { + "conda-package": { + "tags": "_name.#" + } } }, "python-3.8": { "group": "conda-python", - "default": true, "env": { "CM_CONDA_PYTHON_VERSION": "3.8" } diff --git a/cm-mlops/script/get-conda/customize.py b/cm-mlops/script/get-conda/customize.py index d5bd0ef439..d8ef13e343 100644 --- a/cm-mlops/script/get-conda/customize.py +++ b/cm-mlops/script/get-conda/customize.py @@ -13,27 +13,29 @@ def preprocess(i): conda_prefix_name = env.get('CM_CONDA_PREFIX_NAME', '') r = None + file_name = 'conda.exe' if os_info['platform'] == 'windows' else 'conda' if conda_prefix_name == '': - file_name = 'conda.exe' if os_info['platform'] == 'windows' else 'conda' tmp_path = env.get('CM_CONDA_INSTALL_PATH', env.get('CM_TMP_PATH', '')) if tmp_path: - tmp_path+=":" + x = ';' if os_info['platform'] == 'windows' else ':' + tmp_path+=x conda_path = os.path.join(os.path.expanduser("~"), "miniconda3", "bin") if os.path.exists(conda_path): tmp_path += os.path.join(os.path.expanduser("~"), "miniconda3", "bin") env['CM_TMP_PATH'] = tmp_path r = i['automation'].find_artifact({'file_name': file_name, - 'env': env, - 'os_info':os_info, - 'default_path_env_key': 'PATH', - 'detect_version':True, - 'env_path_key':'CM_CONDA_BIN_WITH_PATH', - 'run_script_input':i['run_script_input'], - 'recursion_spaces':recursion_spaces}) + 'env': env, + 'os_info':os_info, + 'default_path_env_key': 'PATH', + 'detect_version':True, + 'env_path_key':'CM_CONDA_BIN_WITH_PATH', + 'run_script_input':i['run_script_input'], + 'recursion_spaces':recursion_spaces}) else: env['CM_CONDA_INSTALL_PATH'] = os.path.join(os.getcwd(), "miniconda3") - env['CM_CONDA_BIN_WITH_PATH'] = os.path.join(env['CM_CONDA_INSTALL_PATH'], "bin", "conda") + bin_dir = 'Scripts' if os_info['platform'] == 'windows' else 'bin' + env['CM_CONDA_BIN_WITH_PATH'] = os.path.join(env['CM_CONDA_INSTALL_PATH'], bin_dir, file_name) if conda_prefix_name != '' or r['return'] >0 : if conda_prefix_name != '' or r['return'] == 16: @@ -47,6 +49,12 @@ def preprocess(i): r = automation.run_native_script({'run_script_input':run_script_input, 'env':env, 'script_name':'install'}) if r['return']>0: return r + # Grigori: temporal fix - should be generalized/improved above + if os_info['platform'] == 'windows' and env.get('CM_CONDA_BIN_WITH_PATH','')=='': + env['CM_CONDA_INSTALL_PATH'] = os.path.join(os.getcwd(), "miniconda3") + env['CM_CONDA_BIN_WITH_PATH'] = os.path.join(env['CM_CONDA_INSTALL_PATH'], 'Scripts', file_name) + + else: found_path = r['found_path'] env['+PATH'] = [ found_path ] diff --git a/cm-mlops/script/get-conda/install.bat b/cm-mlops/script/get-conda/install.bat new file mode 100644 index 0000000000..2528840d9c --- /dev/null +++ b/cm-mlops/script/get-conda/install.bat @@ -0,0 +1,9 @@ +if exist Miniconda3-latest-Windows-x86_64.exe ( + del /Q /S Miniconda3-latest-Windows-x86_64.exe +) + +wget --no-check-certificate https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe +IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% + +start /wait "" Miniconda3-latest-Windows-x86_64.exe /InstallationType=JustMe /RegisterPython=0 /S /D=%CD%\miniconda3 +IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% diff --git a/cm-mlops/script/get-conda/run.bat b/cm-mlops/script/get-conda/run.bat new file mode 100644 index 0000000000..99b9d97d27 --- /dev/null +++ b/cm-mlops/script/get-conda/run.bat @@ -0,0 +1 @@ +%CM_CONDA_BIN_WITH_PATH% --version > tmp-ver.out diff --git a/cm-mlops/script/get-dataset-cnndm/_cm.json b/cm-mlops/script/get-dataset-cnndm/_cm.json index fd70b9d6b9..36af72cdc5 100644 --- a/cm-mlops/script/get-dataset-cnndm/_cm.json +++ b/cm-mlops/script/get-dataset-cnndm/_cm.json @@ -16,16 +16,17 @@ "python", "python3" ], - "tags": "get,python3" + "tags": "get,python3", + "version_max": "3.9.999" }, { - "force_env_keys": [ - "CM_GIT_*" - ], "names": [ "inference-src" ], - "tags": "mlperf,inference,source" + "tags": "mlperf,inference,source", + "skip_if_env": { + "CM_CNNDM_INTEL_VARIATION": [ "yes" ] + } }, { "tags": "get,generic-python-lib,_package.simplejson" @@ -43,11 +44,6 @@ "env": { "CM_DATASET": "CNNDM" }, - "new_env_keys": [ - "CM_DATASET_PATH", - "CM_DATASET_EVAL_PATH", - "CM_CALIBRATION_DATASET_PATH" - ], "tags": [ "get", "dataset", @@ -59,17 +55,33 @@ "uid": "aed298c156e24257", "variations": { "calibration": { + "new_env_keys": [ + "CM_CALIBRATION_DATASET_PATH", + "CM_CALIBRATION_DATASET_CNNDM_PATH" + ], "env": { "CM_DATASET_CALIBRATION": "yes" }, "group": "dataset-type" }, "validation": { + "new_env_keys": [ + "CM_DATASET_PATH", + "CM_DATASET_EVAL_PATH", + "CM_DATASET_CNNDM_EVAL_PATH" + ], "default": true, "env": { "CM_DATASET_CALIBRATION": "no" }, "group": "dataset-type" + }, + "intel": { + }, + "intel,validation": { + "env": { + "CM_CNNDM_INTEL_VARIATION": "yes" + } } } } diff --git a/cm-mlops/script/get-dataset-cnndm/customize.py b/cm-mlops/script/get-dataset-cnndm/customize.py index d7ca59f20c..27363d8000 100644 --- a/cm-mlops/script/get-dataset-cnndm/customize.py +++ b/cm-mlops/script/get-dataset-cnndm/customize.py @@ -6,7 +6,10 @@ def preprocess(i): env = i['env'] - print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") + if env.get('CM_CNNDM_INTEL_VARIATION', '') == 'yes': + i['run_script_input']['script_name'] = "run-intel" + else: + print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") return {'return': 0} @@ -16,7 +19,11 @@ def postprocess(i): if env.get('CM_DATASET_CALIBRATION','') == "no": env['CM_DATASET_PATH'] = os.path.join(os.getcwd(), 'install') env['CM_DATASET_EVAL_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_eval.json') + env['CM_DATASET_CNNDM_EVAL_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_eval.json') + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_DATASET_PATH'] else: env['CM_CALIBRATION_DATASET_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_dailymail_calibration.json') + env['CM_CALIBRATION_DATASET_CNNDM_PATH'] = os.path.join(os.getcwd(), 'install', 'cnn_dailymail_calibration.json') + env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_CALIBRATION_DATASET_PATH'] return {'return': 0} diff --git a/cm-mlops/script/get-dataset-cnndm/run-intel.sh b/cm-mlops/script/get-dataset-cnndm/run-intel.sh new file mode 100644 index 0000000000..067f158a56 --- /dev/null +++ b/cm-mlops/script/get-dataset-cnndm/run-intel.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +CUR=${PWD} +rm -rf install +mkdir -p install + +export DATASET_CNNDM_PATH=${CUR}/install + +wget -nc https://raw.githubusercontent.com/mlcommons/inference_results_v3.1/main/closed/Intel/code/gptj-99/pytorch-cpu/download-dataset.py +test $? -eq 0 || exit 1 + +cmd="${CM_PYTHON_BIN_WITH_PATH} download-dataset.py --split validation --output-dir ${DATASET_CNNDM_PATH}" +echo "$cmd" +eval "$cmd" +test $? -eq 0 || exit 1 diff --git a/cm-mlops/script/get-dataset-coco/README-extra.md b/cm-mlops/script/get-dataset-coco/README-extra.md index 5dfa1885bc..9f19d2e8d7 100644 --- a/cm-mlops/script/get-dataset-coco/README-extra.md +++ b/cm-mlops/script/get-dataset-coco/README-extra.md @@ -92,3 +92,4 @@ It is useful when all files are already downloaded and saved for common use. ```bash cmr "get coco dataset _val _2017" --to=d:\Downloads\COCO-2017-val --store=d:\Downloads +``` diff --git a/cm-mlops/script/get-dataset-coco/_cm.json b/cm-mlops/script/get-dataset-coco/_cm.json index 883e0b5363..a61234ad0c 100644 --- a/cm-mlops/script/get-dataset-coco/_cm.json +++ b/cm-mlops/script/get-dataset-coco/_cm.json @@ -17,7 +17,8 @@ "input_mapping": { "from": "CM_FROM", "store": "CM_STORE", - "to": "CM_TO" + "to": "CM_TO", + "home": "CM_HOME_DIR" }, "new_env_keys": [ "CM_DATASET_COCO*", @@ -36,6 +37,7 @@ "get-dataset-coco-data", "746e5dad5e784ad6" ], + "skip_if_fake_run": true, "skip_if_env": { "CM_DATASET_COCO_DETECTED": [ "yes" @@ -61,6 +63,7 @@ "get-dataset-coco-annotations", "edb6cd092ff64171" ], + "skip_if_fake_run": true, "skip_if_env": { "CM_DATASET_COCO_DETECTED": [ "yes" @@ -116,5 +119,19 @@ }, "group": "type" } + }, + "docker": { + "skip_run_cmd":"no", + "mount_current_dir":"yes", + "input_paths": [ + "store", + "from", + "to" + ], + "skip_input_for_fake_run": [ + "store", + "from", + "to" + ] } } diff --git a/cm-mlops/script/get-dataset-openimages/README.extra.md b/cm-mlops/script/get-dataset-openimages/README.extra.md new file mode 100644 index 0000000000..b6f5d0812b --- /dev/null +++ b/cm-mlops/script/get-dataset-openimages/README.extra.md @@ -0,0 +1,2 @@ +# Ubuntu 22.04 +`sudo apt-get install -y libgl1-mesa-dev` diff --git a/cm-mlops/script/get-dataset-openimages/customize.py b/cm-mlops/script/get-dataset-openimages/customize.py index 03bab05356..3040fff2ba 100644 --- a/cm-mlops/script/get-dataset-openimages/customize.py +++ b/cm-mlops/script/get-dataset-openimages/customize.py @@ -4,14 +4,64 @@ def preprocess(i): + os_info = i['os_info'] env = i['env'] - print("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") + print ("") + print ("Using MLCommons Inference source from '" + env['CM_MLPERF_INFERENCE_SOURCE'] +"'") + print ("") + + if os_info['platform'] == 'windows': + MLPERF_CLASSES=['Airplane','Antelope','Apple','Backpack','Balloon','Banana', + 'Barrel','Baseball bat','Baseball glove','Bee','Beer','Bench','Bicycle', + 'Bicycle helmet','Bicycle wheel','Billboard','Book','Bookcase','Boot', + 'Bottle','Bowl','Bowling equipment','Box','Boy','Brassiere','Bread', + 'Broccoli','Bronze sculpture','Bull','Bus','Bust','Butterfly','Cabinetry', + 'Cake','Camel','Camera','Candle','Candy','Cannon','Canoe','Carrot','Cart', + 'Castle','Cat','Cattle','Cello','Chair','Cheese','Chest of drawers','Chicken', + 'Christmas tree','Coat','Cocktail','Coffee','Coffee cup','Coffee table','Coin', + 'Common sunflower','Computer keyboard','Computer monitor','Convenience store', + 'Cookie','Countertop','Cowboy hat','Crab','Crocodile','Cucumber','Cupboard', + 'Curtain','Deer','Desk','Dinosaur','Dog','Doll','Dolphin','Door','Dragonfly', + 'Drawer','Dress','Drum','Duck','Eagle','Earrings','Egg (Food)','Elephant', + 'Falcon','Fedora','Flag','Flowerpot','Football','Football helmet','Fork', + 'Fountain','French fries','French horn','Frog','Giraffe','Girl','Glasses', + 'Goat','Goggles','Goldfish','Gondola','Goose','Grape','Grapefruit','Guitar', + 'Hamburger','Handbag','Harbor seal','Headphones','Helicopter','High heels', + 'Hiking equipment','Horse','House','Houseplant','Human arm','Human beard', + 'Human body','Human ear','Human eye','Human face','Human foot','Human hair', + 'Human hand','Human head','Human leg','Human mouth','Human nose','Ice cream', + 'Jacket','Jeans','Jellyfish','Juice','Kitchen & dining room table','Kite', + 'Lamp','Lantern','Laptop','Lavender (Plant)','Lemon','Light bulb','Lighthouse', + 'Lily','Lion','Lipstick','Lizard','Man','Maple','Microphone','Mirror', + 'Mixing bowl','Mobile phone','Monkey','Motorcycle','Muffin','Mug','Mule', + 'Mushroom','Musical keyboard','Necklace','Nightstand','Office building', + 'Orange','Owl','Oyster','Paddle','Palm tree','Parachute','Parrot','Pen', + 'Penguin','Personal flotation device','Piano','Picture frame','Pig','Pillow', + 'Pizza','Plate','Platter','Porch','Poster','Pumpkin','Rabbit','Rifle', + 'Roller skates','Rose','Salad','Sandal','Saucer','Saxophone','Scarf','Sea lion', + 'Sea turtle','Sheep','Shelf','Shirt','Shorts','Shrimp','Sink','Skateboard', + 'Ski','Skull','Skyscraper','Snake','Sock','Sofa bed','Sparrow','Spider','Spoon', + 'Sports uniform','Squirrel','Stairs','Stool','Strawberry','Street light', + 'Studio couch','Suit','Sun hat','Sunglasses','Surfboard','Sushi','Swan', + 'Swimming pool','Swimwear','Tank','Tap','Taxi','Tea','Teddy bear','Television', + 'Tent','Tie','Tiger','Tin can','Tire','Toilet','Tomato','Tortoise','Tower', + 'Traffic light','Train','Tripod','Truck','Trumpet','Umbrella','Van','Vase', + 'Vehicle registration plate','Violin','Wall clock','Waste container','Watch', + 'Whale','Wheel','Wheelchair','Whiteboard','Window','Wine','Wine glass','Woman', + 'Zebra','Zucchini'] + + x = '' + for v in MLPERF_CLASSES: + if x!='': x+=' ' + x+='"'+v+'"' + env['CM_DATASET_OPENIMAGES_CLASSES']=x return {'return': 0} def postprocess(i): env = i['env'] + env['CM_DATASET_ANNOTATIONS_DIR_PATH'] = os.path.join(os.getcwd(), 'install', 'annotations') if env.get('CM_DATASET_CALIBRATION','') == "no": diff --git a/cm-mlops/script/get-dataset-openimages/run.bat b/cm-mlops/script/get-dataset-openimages/run.bat index 46ee3ebd6d..3b1b6a15e8 100644 --- a/cm-mlops/script/get-dataset-openimages/run.bat +++ b/cm-mlops/script/get-dataset-openimages/run.bat @@ -15,5 +15,8 @@ if not "%CM_DATASET_SIZE%" == "" ( set MAX_IMAGES= ) -%CM_PYTHON_BIN% tools\openimages.py %MAX_IMAGES% --dataset-dir=%INSTALL_DIR% --output-labels=openimages-mlperf.json +%CM_PYTHON_BIN% tools\openimages.py %MAX_IMAGES% --dataset-dir=%INSTALL_DIR% --output-labels=openimages-mlperf.json --classes %CM_DATASET_OPENIMAGES_CLASSES% IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% + +cd %INSTALL_DIR% +move validation\data\* . \ No newline at end of file diff --git a/cm-mlops/script/get-generic-python-lib/_cm.json b/cm-mlops/script/get-generic-python-lib/_cm.json index 7f1909d214..88ab4ce47c 100644 --- a/cm-mlops/script/get-generic-python-lib/_cm.json +++ b/cm-mlops/script/get-generic-python-lib/_cm.json @@ -73,6 +73,16 @@ "CM_TMP_PYTHON_PACKAGE_FORCE_INSTALL": "yes" } }, + "conda.#": { + "ad": { + "python-pip": { + "tags": "_conda.#" + }, + "python3": { + "tags": "_conda.#" + } + } + }, "custom-python": { "env": { "CM_TMP_USE_CUSTOM_PYTHON": "on" diff --git a/cm-mlops/script/get-generic-python-lib/customize.py b/cm-mlops/script/get-generic-python-lib/customize.py index 5027d82376..c0d5b4334f 100644 --- a/cm-mlops/script/get-generic-python-lib/customize.py +++ b/cm-mlops/script/get-generic-python-lib/customize.py @@ -16,10 +16,12 @@ def preprocess(i): if package_name == "onnxruntime_gpu": # https://onnxruntime.ai/docs/execution-providers/CUDA-ExecutionProvider.html#requirements - cuda_version_split = env['CM_CUDA_VERSION'].split(".") - if int(cuda_version_split[0]) >= 12: - # env['CM_INSTALL_ONNXRUNTIME_GPU_FROM_SRC'] = "yes" - return {'return': 1, 'error':'at this moment, onnxruntime_gpu needs CUDA < 12'} + cuda_version = env.get('CM_CUDA_VERSION','').strip() + if cuda_version!='': + cuda_version_split = cuda_version.split('.') + if int(cuda_version_split[0]) >= 12: + # env['CM_INSTALL_ONNXRUNTIME_GPU_FROM_SRC'] = "yes" + return {'return': 1, 'error':'at this moment, PIP package "onnxruntime_gpu" needs CUDA < 12'} extra = env.get('CM_GENERIC_PYTHON_PIP_EXTRA','') if (pip_version and len(pip_version) > 1 and int(pip_version[0]) >= 23) and ('--break-system-packages' not in extra): diff --git a/cm-mlops/script/get-generic-python-lib/run.sh b/cm-mlops/script/get-generic-python-lib/run.sh index 0d3ea80052..5a2a4b56bb 100644 --- a/cm-mlops/script/get-generic-python-lib/run.sh +++ b/cm-mlops/script/get-generic-python-lib/run.sh @@ -3,5 +3,5 @@ CM_TMP_CURRENT_SCRIPT_PATH=${CM_TMP_CURRENT_SCRIPT_PATH:-$PWD} cmd="${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/detect-version.py > tmp-ver.out 2> tmp-ver.err" -echo $cmd +#echo $cmd eval $cmd diff --git a/cm-mlops/script/get-ml-model-gptj/_cm.json b/cm-mlops/script/get-ml-model-gptj/_cm.json index f5be804c33..51f6cf3bb1 100644 --- a/cm-mlops/script/get-ml-model-gptj/_cm.json +++ b/cm-mlops/script/get-ml-model-gptj/_cm.json @@ -53,7 +53,6 @@ } }, "fp32": { - "default": true, "env": { "CM_ML_MODEL_INPUT_DATA_TYPES": "fp32", "CM_ML_MODEL_PRECISION": "fp32", @@ -92,6 +91,13 @@ } } }, + "int4": { + "env": { + "CM_ML_MODEL_INPUT_DATA_TYPES": "int4", + "CM_ML_MODEL_WEIGHT_DATA_TYPES": "int4" + }, + "group": "precision" + }, "uint8": { "env": { "CM_ML_MODEL_INPUT_DATA_TYPES": "uint8", @@ -99,6 +105,78 @@ "CM_ML_MODEL_WEIGHT_DATA_TYPES": "uint8" }, "group": "precision" + }, + "mlcommons": { + "default_variations": { + "precision": "fp32" + }, + "group": "model-provider", + "default": true + }, + "intel": { + "default_variations": { + "framework": "pytorch" + }, + "group": "model-provider" + }, + "pytorch,intel": { + "default_variations": { + "precision": "int8" + }, + "adr": { + "conda-package": { + "tags": "_name.gptj-pt" + } + }, + "deps": [ + { + "tags": "get,mlperf,inference,results", + "version": "v3.1" + }, + { + "tags": "get,ml-model,gpt-j,_fp32,_pytorch", + "env": { + "CM_GPTJ_INTEL_MODEL": "" + }, + "force_new_env_keys": [ + "GPTJ_CHECKPOINT_PATH" + ] + }, + { + "tags": "get,conda,_name.gptj-pt" + }, + { + "tags": "get,python,_conda.gptj-pt" + }, + { + "names": [ + "conda-package", + "intel-openmp" + ], + "tags": "get,generic,conda-package,_package.intel-openmp,_source.intel", + "version": "2023.1.0" + }, + { + "tags": "get,generic,conda-package,_package.jemalloc,_source.conda-forge", + "names": [ + "conda-package", + "jemalloc" + ] + }, + { + "tags": "install,ipex,from.src,_for-intel-mlperf-inference-v3.1-gptj" + }, + { + "tags": "get,dataset,cnndm,_calibration" + } + ], + "env": { + "CM_GPTJ_INTEL_MODEL": "yes" + } + }, + "pytorch,int8,intel": { + }, + "pytorch,int4,intel": { } } } diff --git a/cm-mlops/script/get-ml-model-gptj/customize.py b/cm-mlops/script/get-ml-model-gptj/customize.py index 1b1b7dd249..36655d6c34 100644 --- a/cm-mlops/script/get-ml-model-gptj/customize.py +++ b/cm-mlops/script/get-ml-model-gptj/customize.py @@ -6,10 +6,26 @@ def preprocess(i): os_info = i['os_info'] env = i['env'] - path = env.get('GPTJ_CHECKPOINT_PATH', '').strip() + if env.get('CM_GPTJ_INTEL_MODEL', '') == 'yes': + i['run_script_input']['script_name'] = 'run-intel' + harness_root = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'code', 'gptj-99', 'pytorch-cpu') + print(f"Harness Root: {harness_root}") + env['CM_HARNESS_CODE_ROOT'] = harness_root + env['CM_CALIBRATION_CODE_ROOT'] = os.path.join(env['CM_MLPERF_INFERENCE_RESULTS_PATH'], 'closed', 'Intel', 'calibration') - if path == '' or not os.path.exists(path): - env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' + env['CHECKPOINT_DIR'] = env['GPTJ_CHECKPOINT_PATH'] + + env['QUANTIZED_MODEL_DIR'] = os.getcwd() + + if env['CM_ML_MODEL_WEIGHT_DATA_TYPES'] == "int8": + env['INT8_MODEL_DIR'] = os.getcwd() + else: + env['INT4_MODEL_DIR'] = os.getcwd() + else: + path = env.get('GPTJ_CHECKPOINT_PATH', '').strip() + + if path == '' or not os.path.exists(path): + env['CM_TMP_REQUIRE_DOWNLOAD'] = 'yes' return {'return':0} @@ -17,7 +33,9 @@ def postprocess(i): env = i['env'] - env['GPTJ_CHECKPOINT_PATH'] = os.path.join(env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final") + if os.path.exists(os.path.join(env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final")): + env['GPTJ_CHECKPOINT_PATH'] = os.path.join(env['GPTJ_CHECKPOINT_PATH'], "checkpoint-final") + env['CM_ML_MODEL_FILE_WITH_PATH'] = env['GPTJ_CHECKPOINT_PATH'] env['CM_ML_MODEL_FILE'] = os.path.basename(env['CM_ML_MODEL_FILE_WITH_PATH']) env['CM_GET_DEPENDENT_CACHED_PATH'] = env['CM_ML_MODEL_FILE_WITH_PATH'] diff --git a/cm-mlops/script/get-ml-model-gptj/run-int4-calibration.sh b/cm-mlops/script/get-ml-model-gptj/run-int4-calibration.sh new file mode 100644 index 0000000000..45c3669e56 --- /dev/null +++ b/cm-mlops/script/get-ml-model-gptj/run-int4-calibration.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export PATH=${CM_CONDA_BIN_PATH}:$PATH + +echo ${CM_CALIBRATION_CODE_ROOT} +cd ${CM_CALIBRATION_CODE_ROOT}/gpt-j/pytorch-cpu/INT4 +pip install -r requirements.txt +bash run_calibration_int4.sh + +test $? -eq 0 || exit $? diff --git a/cm-mlops/script/get-ml-model-gptj/run-intel.sh b/cm-mlops/script/get-ml-model-gptj/run-intel.sh new file mode 100644 index 0000000000..f6cb2134da --- /dev/null +++ b/cm-mlops/script/get-ml-model-gptj/run-intel.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +export PATH=${CM_CONDA_BIN_PATH}:$PATH + +export CALIBRATION_DATA_JSON=${CM_CALIBRATION_DATASET_CNNDM_PATH} + + +if [[ ${CM_ML_MODEL_WEIGHT_DATA_TYPES} == "int4" ]]; then + export INT4_CALIBRATION_DIR=${PWD}/quantized-int4-model + bash ${CM_TMP_CURRENT_SCRIPT_PATH}/run-int4-calibration.sh + cd ${CM_HARNESS_CODE_ROOT} + bash run_quantization_int4.sh +else + cd ${CM_HARNESS_CODE_ROOT} + bash run_quantization.sh +fi + +test $? -eq 0 || exit $? diff --git a/cm-mlops/script/get-python3/_cm.json b/cm-mlops/script/get-python3/_cm.json index 5ab8b7defb..85066e694d 100644 --- a/cm-mlops/script/get-python3/_cm.json +++ b/cm-mlops/script/get-python3/_cm.json @@ -41,6 +41,30 @@ ], "uid": "d0b5dd74373f4a62", "variations": { + "custom-path.#": { + "env": { + "CM_PYTHON_BIN_WITH_PATH": "#" + } + }, + "conda.#": { + "deps": [ + { + "tags": "get,generic,conda-package,_name.#,_package.python", + "names": [ + "conda-package", + "conda-python" + ] + } + ], + "adr": { + "pip-package": { + "tags": "_conda.#" + } + }, + "env": { + "CM_PYTHON_CONDA": "yes" + } + }, "lto": {}, "optimized": {}, "shared": {}, diff --git a/cm-mlops/script/get-python3/customize.py b/cm-mlops/script/get-python3/customize.py index 11373dba80..3e2cc0b6f4 100644 --- a/cm-mlops/script/get-python3/customize.py +++ b/cm-mlops/script/get-python3/customize.py @@ -7,6 +7,9 @@ def preprocess(i): env = i['env'] + if env.get('CM_PYTHON_CONDA', '') == 'yes': + env['CM_PYTHON_BIN_WITH_PATH'] = os.path.join(env['CM_CONDA_BIN_PATH'], "python") + recursion_spaces = i['recursion_spaces'] # we need to understand whether this script is called first and CM_PYTHON_BIN_WITH_PATH is empty diff --git a/cm-mlops/script/gui/app.py b/cm-mlops/script/gui/app.py index 49546be600..70e81b548f 100644 --- a/cm-mlops/script/gui/app.py +++ b/cm-mlops/script/gui/app.py @@ -217,6 +217,7 @@ def main(): tags = '--tags={}'.format(x) # Check flags + flags_dict = {} flags = '' for key in st_inputs: value = st_inputs[key] @@ -224,19 +225,30 @@ def main(): if value!='' and (type(value)!=bool or value==True): flags+=' '+var1+'\n --'+key2 + + z = True if type(value)!=bool: x = str(value) + z = x + if ' ' in x or ':' in x or '/' in x or '\\' in x: x='"'+x+'"' flags+='='+x + flags_dict[key2]=z + ######################################################## # Extra CMD st.markdown("""---""") cmd_extension = st.text_input("Add extra to CM script command line").strip() + run_via_docker = False + if len(meta.get('docker',{}))>0: + run_via_docker = st.checkbox('Run via Docker', key='run_via_docker', value=False) + # Prepare CLI - cli = 'cm run script {} {} '.format(tags, flags,) + action = 'docker' if run_via_docker else 'run' + cli = 'cm {} script {} {} '.format(action, tags, flags,) x = '' if cmd_extension=='' else var1+'\n '+cmd_extension @@ -248,6 +260,7 @@ def main(): # if no_run=='': # cli+=' --pause\n' + # Print CLI st.markdown("""---""") @@ -278,7 +291,44 @@ def main(): st.text_area('**Install [CM interface](https://github.com/mlcommons/ck) with a few dependencies:**', x, height=170) - cli = st.text_area('**Run CM script:**', cli, height=500) + + st.markdown("**Run CM script from Python:**") + + # Python API + dd = { + 'action':action, + 'automation':'script,5b4e0237da074764', + 'tags':script_tags + } + + x = script_tags + if len(selected_variations)>0: + for sv in selected_variations: + x+=' '+sv + + dd['tags']=x.replace(' ',',') + + dd.update(flags_dict) + + import json + dd_json=json.dumps(dd, indent=2) + + y = 'import cmind\n' + y+= 'r = cmind.access('+dd_json+')\n' + y+= 'if r[\'return\']>0: print (r)\n' + + x=''' + ```python + {} + '''.format(y) + +# st.write(x.replace('\n','
\n'), unsafe_allow_html=True) + + st.markdown(x) + + + cli = st.text_area('**Run CM script from command line:**', cli, height=500) + # Add explicit button "Run" diff --git a/cm-mlops/script/install-llvm-prebuilt/README-extra.md b/cm-mlops/script/install-llvm-prebuilt/README-extra.md index 34c4385c4d..1ad1e122be 100644 --- a/cm-mlops/script/install-llvm-prebuilt/README-extra.md +++ b/cm-mlops/script/install-llvm-prebuilt/README-extra.md @@ -13,6 +13,7 @@ This [CM script](https://github.com/mlcommons/ck/blob/master/cm/docs/specs/scrip ## Supported and Tested OS 1. Ubuntu 18.04, 20.04, 22.04 2. RHEL 9 +3. Windows 10, 11 # CLI @@ -58,6 +59,19 @@ cm run script "app image corner-detection" * Windows, Linux, MacOS +### Ubuntu 22.04 + +* 17.0.6 +* 17.0.5 +* 17.0.4 +* 17.0.2 +* 16.0.4 +* 16.0.0 + `sudo apt install libncurses5` +* 15.0.6 +* 14.0.0 + + ### RHEL 9 #### v14.0.0: ✓ diff --git a/cm-mlops/script/install-llvm-prebuilt/_cm.json b/cm-mlops/script/install-llvm-prebuilt/_cm.json index 4db0625ab4..39cef74498 100644 --- a/cm-mlops/script/install-llvm-prebuilt/_cm.json +++ b/cm-mlops/script/install-llvm-prebuilt/_cm.json @@ -4,7 +4,7 @@ "category": "Compiler automation", "automation_uid": "5b4e0237da074764", "cache": true, - "default_version": "14.0.0", + "default_version": "15.0.6", "deps": [ { "tags": "detect,os" diff --git a/cm-mlops/script/install-llvm-prebuilt/customize.py b/cm-mlops/script/install-llvm-prebuilt/customize.py index f2ac97f67b..25cd6b922f 100644 --- a/cm-mlops/script/install-llvm-prebuilt/customize.py +++ b/cm-mlops/script/install-llvm-prebuilt/customize.py @@ -61,7 +61,22 @@ def preprocess(i): # Treat all Linux flavours as Ubuntu for now ... if True: - default_os='18.04' + need_version_split = need_version.split('.') + + default_os = '22.04' + + if len(need_version_split)>0: + hver = 0 + try: + hver = int(need_version_split[0]) + except: + pass + + if hver>0: + if hver<16: + default_os='18.04' + else: + default_os='22.04' if need_version == '10.0.1': default_os = '16.04' @@ -95,7 +110,17 @@ def preprocess(i): elif need_version == '14.0.0': default_os = '18.04' - package_name = 'clang+llvm-' + need_version + '-x86_64-linux-gnu-ubuntu-' + default_os + '.tar.xz' + elif need_version == '15.0.6': + default_os = '18.04' + + elif need_version == '16.0.0': + default_os = '18.04' + + elif need_version == '16.0.4': + default_os = '22.04' + + + package_name = 'clang+llvm-' + need_version + '-x86_64-linux-gnu-ubuntu-' + default_os + '.tar.xz' package_url = 'https://github.com/llvm/llvm-project/releases/download/llvmorg-' + need_version + '/' + package_name diff --git a/cm-mlops/script/print-hello-world-py/_cm.json b/cm-mlops/script/print-hello-world-py/_cm.json index 818760ded1..5e6a0629c7 100644 --- a/cm-mlops/script/print-hello-world-py/_cm.json +++ b/cm-mlops/script/print-hello-world-py/_cm.json @@ -18,7 +18,11 @@ "tags": "get,python3" }, { - "tags": "print,python-version" + "tags": "print,python-version", + "skip_if_env": { + "CM_SKIP_PRINT":["True"], + "CM_SKIP_PRINT2":["True"] + } } ], "tags": [ diff --git a/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml b/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml index 49607eff8f..5011cec210 100644 --- a/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml +++ b/cm-mlops/script/reproduce-mlperf-inference-intel/_cm.yaml @@ -138,6 +138,8 @@ deps: # Install MLPerf inference dependencies - tags: get,mlperf,inference,results + names: + inference-results version: v3.1 @@ -253,13 +255,15 @@ variations: CM_BENCHMARK: STANDALONE_GPTJ gptj_,pytorch: + adr: + conda-package: + tags: _name.gptj-pt deps: - tags: get,conda,_name.gptj-pt - - tags: get,generic,conda-package,_package.python - names: - - conda-package - - python - version: "3.9" + - tags: get,python,_conda.gptj-pt + adr: + conda-python: + version: "3.9" - tags: install,llvm,src,_tag.llvmorg-16.0.6,_clang,_release,_for-intel-mlperf-inference-v3.1-gptj - names: - conda-package @@ -271,9 +275,9 @@ variations: - conda-package - jemalloc - tags: install,ipex,from.src,_for-intel-mlperf-inference-v3.1-gptj - - tags: get,generic-python-lib,_custom-python,_package.transformers - env: - CM_PYTHON_BIN_WITH_PATH: "<<>>/python3" + - tags: get,generic-python-lib,_package.transformers + names: + - pip-package version: "4.28.1" - tags: get,mlcommons,inference,src names: @@ -288,12 +292,12 @@ variations: - ml-model - gptj-model - gpt-j-model - - tags: get,generic-python-lib,_custom-python,_package.datasets - env: - CM_PYTHON_BIN_WITH_PATH: "<<>>/python3" - - tags: get,generic-python-lib,_custom-python,_package.accelerate - env: - CM_PYTHON_BIN_WITH_PATH: "<<>>/python3" + - tags: get,generic-python-lib,_package.datasets + names: + - pip-package + - tags: get,generic-python-lib,_package.accelerate + names: + - pip-package gptj-99: group: model diff --git a/cm-mlops/script/run-mlperf-inference-app/customize.py b/cm-mlops/script/run-mlperf-inference-app/customize.py index eb96904b14..56a6624c44 100644 --- a/cm-mlops/script/run-mlperf-inference-app/customize.py +++ b/cm-mlops/script/run-mlperf-inference-app/customize.py @@ -98,7 +98,8 @@ def preprocess(i): test_list = ["TEST01", "TEST05"] if env['CM_MODEL'] in ["resnet50"]: test_list.append("TEST04") - if "gpt" in env['CM_MODEL']: + if "gpt" in env['CM_MODEL'] or "sdxl" in env['CM_MODEL'] or "llama2-70b" in env['CM_MODEL']: + test_list.remove("TEST01") test_list.remove("TEST05") variation_implementation= "_" + env.get("CM_MLPERF_IMPLEMENTATION", "reference") diff --git a/cm/README.md b/cm/README.md index 99e6e2ccf3..70d9c1c781 100644 --- a/cm/README.md +++ b/cm/README.md @@ -17,7 +17,7 @@ either natively or inside containers. Here are a few most commonly used examples from the CM users that should run in the same way on Linux, MacOS, Windows and other platforms -(see [Gettings Started Guide](docs/getting-started.md) to understand +(see [Getting Started Guide](docs/getting-started.md) to understand how they work and how to reuse them in your projects): ```bash @@ -30,6 +30,7 @@ cmr "python app image-classification onnx" cmr "download file _wget" --url=https://cKnowledge.org/ai/data/computer_mouse.jpg --verify=no --env.CM_DOWNLOAD_CHECKSUM=45ae5c940233892c2f860efdf0b66e7e cmr "python app image-classification onnx" --input=computer_mouse.jpg +cmr "python app image-classification onnx" --input=computer_mouse.jpg --debug cm find script "python app image-classification onnx" @@ -38,7 +39,8 @@ cmr "install python-venv" --version_max=3.10.11 --name=mlperf cmr "get ml-model stable-diffusion" cmr "get ml-model huggingface zoo _model-stub.alpindale/Llama-2-13b-ONNX" --model_filename=FP32/LlamaV2_13B_float32.onnx --skip_cache -cmr "get coco dataset _val _2014" +cmr "get dataset coco _val _2014" +cmr "get dataset openimages" -j cm show cache cm show cache "get ml-model stable-diffusion" @@ -49,17 +51,29 @@ cmr "python app image-classification onnx" --input=computer_mouse.jpg cm rm cache -f cmr "python app image-classification onnx" --input=computer_mouse.jpg --adr.onnxruntime.version_max=1.16.0 - +cmr "get cuda" --version_min=12.0.0 --version_max=12.3.1 cmr "python app image-classification onnx _cuda" --input=computer_mouse.jpg -cmr "cm gui" --script="python app image-classification onnx" +cm gui script "python app image-classification onnx" cm docker script "python app image-classification onnx" --input=computer_mouse.jpg cm docker script "python app image-classification onnx" --input=computer_mouse.jpg -j -docker_it +cm docker script "get coco dataset _val _2017" --to=d:\Downloads\COCO-2017-val --store=d:\Downloads --docker_cm_repo=ctuning@mlcommons-ck + cmr "run common mlperf inference" --implementation=nvidia --model=bert-99 --category=datacenter --division=closed cm find script "run common mlperf inference" +cmr "get generic-python-lib _package.torch" --version=2.1.1 +cmr "get generic-python-lib _package.torchvision" --version=0.16.2 +cmr "python app image-classification torch" --input=computer_mouse.jpg + + +cmr "install llvm prebuilt" --version=17.0.6 +cmr "app image corner-detection" + +cmr "get conda" + cm pull repo ctuning@cm-reproduce-research-projects cmr "reproduce paper micro-2023 victima _install_deps" cmr "reproduce paper micro-2023 victima _run" diff --git a/platform/README.md b/platform/README.md index 6f8499e1f4..e75de5c89a 100644 --- a/platform/README.md +++ b/platform/README.md @@ -1,6 +1,7 @@ # Collective Knowledge Playground -*Note that this project is under heavy development.* +*Note that this project is under heavy development. + We are preparing a major update based on very useful feedback from our users during MLPerf inference 3.1 community submission!* ### Introduction