From eaf85e61824d6d617918f1ebba6aad360ef404a9 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 9 Feb 2024 20:59:37 +0530 Subject: [PATCH 01/17] dlrm -> dlrm-v2 --- .../script/generate-mlperf-inference-user-conf/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cm-mlops/script/generate-mlperf-inference-user-conf/customize.py b/cm-mlops/script/generate-mlperf-inference-user-conf/customize.py index 7050230012..6f1b8b5b39 100644 --- a/cm-mlops/script/generate-mlperf-inference-user-conf/customize.py +++ b/cm-mlops/script/generate-mlperf-inference-user-conf/customize.py @@ -55,7 +55,7 @@ def preprocess(i): RUN_CMD = "" state['RUN'] = {} test_list = ["TEST01", "TEST04", "TEST05"] - if env['CM_MODEL'] in ["rnnt", "bert-99", "bert-99.9", "dlrm-99", "dlrm-99.9", "3d-unet-99", "3d-unet-99.9"]: + if env['CM_MODEL'] in ["rnnt", "bert-99", "bert-99.9", "dlrm-v2-99", "dlrm-v2-99.9", "3d-unet-99", "3d-unet-99.9"]: test_list.remove("TEST04") if "gpt-" in env['CM_MODEL']: test_list.remove("TEST05") @@ -93,7 +93,7 @@ def preprocess(i): if 'bert' in ml_model_name: ml_model_name = "bert" if 'dlrm' in ml_model_name: - ml_model_name = "dlrm" + ml_model_name = "dlrm-v2" if '3d-unet' in ml_model_name: ml_model_name = "3d-unet" if 'gptj' in ml_model_name: From 798f9d9ba85fcb84195d9909dc495a65d8112365 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Fri, 9 Feb 2024 16:54:16 +0100 Subject: [PATCH 02/17] improving tests --- .../any-model-offline.yaml | 2 +- .../any-model-offline.yaml | 2 +- .../bert-reference-python-onnx-offline.yaml | 2 +- .../gptj-reference-python-torch-offline.yaml | 2 +- cm-mlops/script/gui/playground_howtorun.py | 1 + cm-mlops/script/launch-benchmark/customize.py | 175 +++++++++++------- 6 files changed, 117 insertions(+), 67 deletions(-) diff --git a/cm-mlops/cfg/benchmark-run-loadgen-cpp-dev/any-model-offline.yaml b/cm-mlops/cfg/benchmark-run-loadgen-cpp-dev/any-model-offline.yaml index ac77ab5a39..6f2d290046 100644 --- a/cm-mlops/cfg/benchmark-run-loadgen-cpp-dev/any-model-offline.yaml +++ b/cm-mlops/cfg/benchmark-run-loadgen-cpp-dev/any-model-offline.yaml @@ -1,4 +1,4 @@ -uid: 4df38ed8dd804678 +uid: 125abafe58dc4473 name: "Any model - offline" diff --git a/cm-mlops/cfg/benchmark-run-loadgen-python-dev/any-model-offline.yaml b/cm-mlops/cfg/benchmark-run-loadgen-python-dev/any-model-offline.yaml index ac77ab5a39..aefd1822f4 100644 --- a/cm-mlops/cfg/benchmark-run-loadgen-python-dev/any-model-offline.yaml +++ b/cm-mlops/cfg/benchmark-run-loadgen-python-dev/any-model-offline.yaml @@ -1,4 +1,4 @@ -uid: 4df38ed8dd804678 +uid: db45dcd686854602 name: "Any model - offline" diff --git a/cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/bert-reference-python-onnx-offline.yaml b/cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/bert-reference-python-onnx-offline.yaml index 2b6c3dcab6..b3a9f0dd1c 100644 --- a/cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/bert-reference-python-onnx-offline.yaml +++ b/cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/bert-reference-python-onnx-offline.yaml @@ -1,4 +1,4 @@ -uid: 4df38ed8dd804678 +uid: 9eee8cb06621413a name: "BERT Reference Python ONNX Offline" diff --git a/cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/gptj-reference-python-torch-offline.yaml b/cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/gptj-reference-python-torch-offline.yaml index 2e5ce3ddc1..e3cc190aaf 100644 --- a/cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/gptj-reference-python-torch-offline.yaml +++ b/cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/gptj-reference-python-torch-offline.yaml @@ -1,4 +1,4 @@ -uid: 4df38ed8dd804678 +uid: 53e4028a3b31400d name: "GPT-J Reference Python Torch Offline" diff --git a/cm-mlops/script/gui/playground_howtorun.py b/cm-mlops/script/gui/playground_howtorun.py index d7f9cb87f9..db1d927163 100644 --- a/cm-mlops/script/gui/playground_howtorun.py +++ b/cm-mlops/script/gui/playground_howtorun.py @@ -54,6 +54,7 @@ def page(st, params, action = ''): initialized = True ii = {'streamlit_module': st, + 'params': params, 'meta': external_module_meta, 'skip_title': True} diff --git a/cm-mlops/script/launch-benchmark/customize.py b/cm-mlops/script/launch-benchmark/customize.py index db77a3e1d6..8baf9f2cf8 100644 --- a/cm-mlops/script/launch-benchmark/customize.py +++ b/cm-mlops/script/launch-benchmark/customize.py @@ -46,6 +46,7 @@ def load_cfg(i): prune = i.get('prune',{}) prune_key = prune.get('key', '') + prune_uid = prune.get('uid', '') prune_list = prune.get('list',[]) # Checking individual files inside CM entry @@ -86,7 +87,11 @@ def load_cfg(i): # Check pruning add = True - if prune_key!='' and len(prune_list)>0 and uid not in prune_list: + + if prune_uid!='' and uid != prune_uid: + add = False + + if add and prune_key!='' and len(prune_list)>0 and uid not in prune_list: add = False if add: @@ -106,6 +111,7 @@ def load_cfg(i): ################################################################################## def gui(i): + params = i['params'] st = i['streamlit_module'] meta = i['meta'] gui_meta = meta['gui'] @@ -119,99 +125,142 @@ def gui(i): st.markdown('### {}'.format(title)) + # Check if test uid is specified + uid = '' + x = params.get('uid',['']) + if len(x)>0 and x[0]!='': uid = x[0].strip() + + # Preparing state if 'bench_id' not in st.session_state: st.session_state['bench_id']=0 if 'compute_id' not in st.session_state: st.session_state['compute_id']=0 ############################################################## # Check the first level of benchmarks - r=load_cfg({'tags':'benchmark,run', 'skip_files':True}) + bench_id = 0 + + ii = {'tags':'benchmark,run', 'skip_files':True} + + if uid != '': + ii['skip_files'] = False + ii['prune']={'uid':uid} + + r=load_cfg(ii) if r['return']>0: return r - selection = sorted(r['selection'], key = lambda v: v['name']) - bench_selection = [{'name':''}] + selection + lst = r['selection'] + + test_meta = {} - # Creating compute selector - bench_id = st.selectbox('Select benchmark:', - range(len(bench_selection)), - format_func=lambda x: bench_selection[x]['name'], - index = 0, - key = 'bench') + if uid == '': + selection = sorted(lst, key = lambda v: v['name']) + bench_selection = [{'name':''}] + selection - bench_supported_compute = [] - bench_meta = {} - if bench_id != st.session_state['bench_id']: - bench_meta = bench_selection[bench_id] - bench_supported_compute = bench_meta.get('supported_compute',[]) + bench_id = st.selectbox('Select benchmark:', + range(len(bench_selection)), + format_func=lambda x: bench_selection[x]['name'], + index = 0, + key = 'bench') - urls = bench_meta.get('urls',[]) - if len(urls)>0: - x = '\n' - for u in urls: - name = u['name'] - url = u['url'] + + bench_supported_compute = [] + bench_meta = {} + if bench_id != st.session_state['bench_id']: + bench_meta = bench_selection[bench_id] + bench_supported_compute = bench_meta.get('supported_compute',[]) - x+=' [ [{}]({}) ] '.format(name, url) - x+='\n' - st.markdown(x) + urls = bench_meta.get('urls',[]) + if len(urls)>0: + x = '\n' + for u in urls: + name = u['name'] + url = u['url'] + x+=' [ [{}]({}) ] '.format(name, url) + x+='\n' + st.markdown(x) - ############################################################## - # Check compute - r=load_cfg({'tags':'benchmark,compute', - 'prune':{'key':'supported_compute', 'list':bench_supported_compute}}) - if r['return']>0: return r - selection = sorted(r['selection'], key = lambda v: v['name']) - compute_selection = [{'name':''}] + selection + ############################################################## + # Check compute + r=load_cfg({'tags':'benchmark,compute', + 'prune':{'key':'supported_compute', 'list':bench_supported_compute}}) + if r['return']>0: return r + selection = sorted(r['selection'], key = lambda v: v['name']) + compute_selection = [{'name':''}] + selection - # Creating compute selector - compute_id = st.selectbox('Select target hardware:', - range(len(compute_selection)), - format_func=lambda x: compute_selection[x]['name'], - index = 0, - key = 'compute') - if compute_id!=st.session_state['compute_id']: - st.session_state['compute_id']=compute_id + # Creating compute selector + compute_id = st.selectbox('Select target hardware:', + range(len(compute_selection)), + format_func=lambda x: compute_selection[x]['name'], + index = 0, + key = 'compute') - try: - st.rerun() - except: - st.experimental_rerun() + if compute_id!=st.session_state['compute_id']: + st.session_state['compute_id']=compute_id + try: + st.rerun() + except: + st.experimental_rerun() - ############################################################## - # Check tests - ii = {'tags':'benchmark,run'} - if bench_id>0: - bench_uid = bench_selection[bench_id]['uid'] - ii['artifact']=bench_uid + ############################################################## + # Check tests + ii = {'tags':'benchmark,run'} + if bench_id>0: + bench_uid = bench_selection[bench_id]['uid'] + ii['artifact']=bench_uid - r=load_cfg(ii) - if r['return']>0: return r - selection = sorted(r['selection'], key = lambda v: v['name']) + r=load_cfg(ii) + if r['return']>0: return r + selection = sorted(r['selection'], key = lambda v: v['name']) - test_selection = [{'name':''}] + selection - + # Check how many and prune + if len(selection)>1: + + test_tags = st.text_input('Found {} CM tests. Prune them by tags:'.format(str(len(selection))), value='', key='test_tags').strip() + + + + + + + test_selection = [{'name':''}] + selection + + + + # Creating compute selector + test_id = st.selectbox('Select test:', + range(len(test_selection)), + format_func=lambda x: test_selection[x]['name'], + index = 0, + key = 'test') + + if test_id >0: + test_meta = test_selection[test_id] - # Creating compute selector - test_id = st.selectbox('Select test:', - range(len(test_selection)), - format_func=lambda x: test_selection[x]['name'], - index = 0, - key = 'test') + else: + if len(lst)==0: + st.markdown('CM test with UID "{}" not found!'.format(uid)) + return {'return':0} + elif len(lst)>1: + st.markdown('Warning: More than 1 CM test found with UID "{}" - ambiguity!'.format(uid)) + return {'return':0} - ############################################################## - if test_id >0: - test_meta = test_selection[test_id] + test_meta = lst[0] + ############################################################## + if len(test_meta)>0: + st.markdown('---') + st.markdown(str(test_meta)) + test_path = test_meta['full_path'] test_md = test_meta['full_path'][:-5]+'.md' From 521530349c25b2a3902deb7fccfbe414423bbd64 Mon Sep 17 00:00:00 2001 From: Arjun Date: Fri, 9 Feb 2024 14:22:30 -0800 Subject: [PATCH 03/17] Add llvm compilation flags for qaic-software-kit --- .../script/get-qaic-software-kit/customize.py | 21 +++++++++++++++++++ cm-mlops/script/get-qaic-software-kit/run.sh | 3 +++ 2 files changed, 24 insertions(+) diff --git a/cm-mlops/script/get-qaic-software-kit/customize.py b/cm-mlops/script/get-qaic-software-kit/customize.py index 5bf2da70c6..898e2c095b 100644 --- a/cm-mlops/script/get-qaic-software-kit/customize.py +++ b/cm-mlops/script/get-qaic-software-kit/customize.py @@ -15,6 +15,27 @@ def preprocess(i): quiet = (env.get('CM_QUIET', False) == 'yes') + if env.get('+ CXXFLAGS', []) == []: + env['+ CXXFLAGS'] = [] + if env.get('+ CFLAGS', []) == []: + env['+ CFLAGS'] = [] + + if env.get('CM_LLVM_CLANG_VERSION', '') != '': + clang_version_split = env['CM_LLVM_CLANG_VERSION'].split(".") + clang_major_version = int(clang_version_split[0]) + if clang_major_version >= 16: + env['+ CFLAGS'].append("-Wno-error=unused-but-set-variable") + if clang_major_version >= 13: + env['+ CFLAGS'].append("-Wno-error=unused-const-variable") + env['+ CFLAGS'].append("-Wno-error=unused-but-set-variable") + env['+ CFLAGS'].append("-Wno-error=strict-prototypes") + env['+ CXXFLAGS'].append("-Wno-error=unused-const-variable") + env['+ CXXFLAGS'].append("-Wno-error=unused-variable") + env['+ CXXFLAGS'].append("-Wno-error=unused-private-field") + env['+ CXXFLAGS'].append("-Wno-error=unused-result") + env['+ CXXFLAGS'].append("-Wno-error=string-concatenation") + env['+ CXXFLAGS'].append("-Wno-error=infinite-recursion") + return {'return':0} def postprocess(i): diff --git a/cm-mlops/script/get-qaic-software-kit/run.sh b/cm-mlops/script/get-qaic-software-kit/run.sh index cb727070a2..a00122a358 100644 --- a/cm-mlops/script/get-qaic-software-kit/run.sh +++ b/cm-mlops/script/get-qaic-software-kit/run.sh @@ -4,6 +4,9 @@ function cmake() { ${CM_CMAKE_BIN_WITH_PATH} $@ } +export CC=${CM_C_COMPILER_WITH_PATH} +export CXX=${CM_CXX_COMPILER_WITH_PATH} + export -f cmake cd ${CM_QAIC_SOFTWARE_KIT_PATH} rm -rf build From 57201935a49f1786a9221932cd87bc6f8a0b37e6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 04:23:46 +0530 Subject: [PATCH 04/17] Support ubuntu 23.04+ in cm docker --- cm-mlops/script/build-dockerfile/customize.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cm-mlops/script/build-dockerfile/customize.py b/cm-mlops/script/build-dockerfile/customize.py index 15d52fe1c1..7ac58c1b1d 100644 --- a/cm-mlops/script/build-dockerfile/customize.py +++ b/cm-mlops/script/build-dockerfile/customize.py @@ -111,6 +111,10 @@ def preprocess(i): f.write('RUN '+ get_value(env, config, 'package-manager-get-cmd') + " " + " ".join(get_value(env, config, 'packages')) + EOL) + if env['CM_DOCKER_OS'] == "ubuntu": + if int(env['CM_DOCKER_OS_VERSION'].split('.')[0]) >= 23: + if "--break-system-packages" not in env.get('CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS', ''): + env['CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS'] = " --break-system-packages" pip_extra_flags = env.get('CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS', '') f.write(EOL+'# Install python packages' + EOL) From 1defa2b22dafc91858df757e3e403ec1ab413e3e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 05:45:05 +0530 Subject: [PATCH 05/17] Fix llvm prebuilt link on Ubuntu 20.04 --- cm-mlops/script/install-llvm-prebuilt/customize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cm-mlops/script/install-llvm-prebuilt/customize.py b/cm-mlops/script/install-llvm-prebuilt/customize.py index 25cd6b922f..236518ca0e 100644 --- a/cm-mlops/script/install-llvm-prebuilt/customize.py +++ b/cm-mlops/script/install-llvm-prebuilt/customize.py @@ -96,8 +96,8 @@ def preprocess(i): elif need_version == '12.0.1': default_os = '16.04' - if host_os_version.startswith('18') or host_os_version.startswith('20'): - default_os = '18.04' + #if host_os_version.startswith('18') or host_os_version.startswith('20'): + # default_os = '18.04' elif need_version == '13.0.0': default_os = '16.04' From 4ca005bd6bb29f2b2cab8bede8de2d6f1d56f7c1 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 05:56:46 +0530 Subject: [PATCH 06/17] Fix compiler flags qaic-software-kit --- cm-mlops/script/get-qaic-software-kit/customize.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cm-mlops/script/get-qaic-software-kit/customize.py b/cm-mlops/script/get-qaic-software-kit/customize.py index 898e2c095b..b36b490d1c 100644 --- a/cm-mlops/script/get-qaic-software-kit/customize.py +++ b/cm-mlops/script/get-qaic-software-kit/customize.py @@ -23,12 +23,16 @@ def preprocess(i): if env.get('CM_LLVM_CLANG_VERSION', '') != '': clang_version_split = env['CM_LLVM_CLANG_VERSION'].split(".") clang_major_version = int(clang_version_split[0]) + if clang_major_version >= 17: + env['+ CFLAGS'].append("-Wno-error=c2x-extensions") if clang_major_version >= 16: env['+ CFLAGS'].append("-Wno-error=unused-but-set-variable") + env['+ CXXFLAGS'].append("-Wno-error=unused-but-set-variable") if clang_major_version >= 13: env['+ CFLAGS'].append("-Wno-error=unused-const-variable") env['+ CFLAGS'].append("-Wno-error=unused-but-set-variable") env['+ CFLAGS'].append("-Wno-error=strict-prototypes") + env['+ CFLAGS'].append("-Wno-error=unused-variable") env['+ CXXFLAGS'].append("-Wno-error=unused-const-variable") env['+ CXXFLAGS'].append("-Wno-error=unused-variable") env['+ CXXFLAGS'].append("-Wno-error=unused-private-field") From 2d2424153109ebe272ba3c261fb24570212b4bf6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 06:08:40 +0530 Subject: [PATCH 07/17] Use prebuilt llvm for installation --- cm-mlops/script/get-llvm/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cm-mlops/script/get-llvm/_cm.json b/cm-mlops/script/get-llvm/_cm.json index 1b33cf2048..f75a38fdef 100644 --- a/cm-mlops/script/get-llvm/_cm.json +++ b/cm-mlops/script/get-llvm/_cm.json @@ -35,7 +35,7 @@ ] }, "reuse_version": true, - "tags": "install,llvm" + "tags": "install,llvm,prebuilt" } ], "sort": 100, From 864c47f9555e6d8ccb2e9f036f2d24cb0e54dc49 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 06:25:18 +0530 Subject: [PATCH 08/17] Fix compilation flags qaic-software-kit --- cm-mlops/script/get-qaic-software-kit/customize.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cm-mlops/script/get-qaic-software-kit/customize.py b/cm-mlops/script/get-qaic-software-kit/customize.py index b36b490d1c..fedeaaf763 100644 --- a/cm-mlops/script/get-qaic-software-kit/customize.py +++ b/cm-mlops/script/get-qaic-software-kit/customize.py @@ -23,11 +23,14 @@ def preprocess(i): if env.get('CM_LLVM_CLANG_VERSION', '') != '': clang_version_split = env['CM_LLVM_CLANG_VERSION'].split(".") clang_major_version = int(clang_version_split[0]) + if clang_major_version >= 17: env['+ CFLAGS'].append("-Wno-error=c2x-extensions") + if clang_major_version >= 16: env['+ CFLAGS'].append("-Wno-error=unused-but-set-variable") env['+ CXXFLAGS'].append("-Wno-error=unused-but-set-variable") + if clang_major_version >= 13: env['+ CFLAGS'].append("-Wno-error=unused-const-variable") env['+ CFLAGS'].append("-Wno-error=unused-but-set-variable") @@ -40,6 +43,9 @@ def preprocess(i): env['+ CXXFLAGS'].append("-Wno-error=string-concatenation") env['+ CXXFLAGS'].append("-Wno-error=infinite-recursion") + if clang_major_version == 12: + env['+ CXXFLAGS'].append("-Wno-error=unknown-warning-option") + return {'return':0} def postprocess(i): From 1c297e0157d97ff897bae52a2c79e3181580b674 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 06:49:44 +0530 Subject: [PATCH 09/17] Fix deps order for qac-software-kit --- cm-mlops/script/get-qaic-software-kit/_cm.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cm-mlops/script/get-qaic-software-kit/_cm.json b/cm-mlops/script/get-qaic-software-kit/_cm.json index cc37d94459..0ad099fd64 100644 --- a/cm-mlops/script/get-qaic-software-kit/_cm.json +++ b/cm-mlops/script/get-qaic-software-kit/_cm.json @@ -11,6 +11,15 @@ "qaic-software-git-repo" ] }, + { + "tags": "get,generic,sys-util,_libudev-dev" + }, + { + "tags": "get,generic,sys-util,_libpci-dev" + }, + { + "tags": "get,google,test" + }, { "tags": "get,cmake", "version_min": "3.24.0", @@ -23,15 +32,6 @@ "names": [ "compiler" ] - }, - { - "tags": "get,generic,sys-util,_libudev-dev" - }, - { - "tags": "get,generic,sys-util,_libpci-dev" - }, - { - "tags": "get,google,test" } ], "input_description": {}, From 925d9ec01e88e616fd88a8aab8fff76e69bab188 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 07:04:33 +0530 Subject: [PATCH 10/17] Upgrade google test default version to 1.14 --- cm-mlops/script/get-google-test/_cm.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cm-mlops/script/get-google-test/_cm.json b/cm-mlops/script/get-google-test/_cm.json index 93dce5f54f..a8fd1d45d1 100644 --- a/cm-mlops/script/get-google-test/_cm.json +++ b/cm-mlops/script/get-google-test/_cm.json @@ -13,7 +13,7 @@ } ], "input_description": {}, - "default_version": "1.13.0", + "default_version": "1.14.0", "input_mapping": {}, "new_env_keys": [ "CM_GOOGLE_TEST_SRC_PATH", From b638b879fe98c52b08d4fc0d157053074177f33e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 15:51:39 +0530 Subject: [PATCH 11/17] Support llvm for google-test build --- cm-mlops/script/get-google-test/_cm.json | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/cm-mlops/script/get-google-test/_cm.json b/cm-mlops/script/get-google-test/_cm.json index a8fd1d45d1..ce17db8aa9 100644 --- a/cm-mlops/script/get-google-test/_cm.json +++ b/cm-mlops/script/get-google-test/_cm.json @@ -6,10 +6,16 @@ "category": "Detection or installation of tools and artifacts", "deps": [ { - "tags": "get,cmake" + "tags": "get,cmake", + "names": [ + "cmake" + ] }, { - "tags": "get,gcc" + "tags": "get,compiler", + "names": [ + "compiler" + ] } ], "input_description": {}, @@ -29,7 +35,8 @@ "force_env_keys": [ "CM_GIT_*" ], - "tags": "get,git,repo,_repo.https://github.com/google/googletest.git" + "tags": "get,git,repo,_repo.https://github.com/google/googletest.git", + "extra_cache_tags": "google-test,gtest" } ], "tags": [ @@ -41,6 +48,5 @@ "google" ], "uid": "02945138a5614253", - "variations": {}, "versions": {} } From 63c8864aaada6214c2f734d46977b2dd9755b203 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 10:32:12 +0000 Subject: [PATCH 12/17] Fix cmake run for googletest --- cm-mlops/script/get-google-test/run.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cm-mlops/script/get-google-test/run.sh b/cm-mlops/script/get-google-test/run.sh index 2aa5b05729..c8a9a44253 100644 --- a/cm-mlops/script/get-google-test/run.sh +++ b/cm-mlops/script/get-google-test/run.sh @@ -1,8 +1,16 @@ #!/bin/bash +function cmake() { +${CM_CMAKE_BIN_WITH_PATH} $@ +} + +export CC=${CM_C_COMPILER_WITH_PATH} +export CXX=${CM_CXX_COMPILER_WITH_PATH} + CUR=$PWD mkdir -p install INSTALL_DIR=$CUR/install cd ${CM_GIT_REPO_CHECKOUT_PATH} + mkdir build cd build export MAKEFLAGS=-j${CM_MAKE_CORES} From b3d4871ce7082b37ffabebd7ea00f6e99f03d76d Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 16:14:09 +0530 Subject: [PATCH 13/17] Added extra cache tags for qaic-software-kit --- cm-mlops/script/get-qaic-software-kit/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cm-mlops/script/get-qaic-software-kit/_cm.json b/cm-mlops/script/get-qaic-software-kit/_cm.json index 0ad099fd64..beffb2ecc9 100644 --- a/cm-mlops/script/get-qaic-software-kit/_cm.json +++ b/cm-mlops/script/get-qaic-software-kit/_cm.json @@ -9,7 +9,8 @@ "tags": "get,git,repo,_repo.https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100", "names": [ "qaic-software-git-repo" - ] + ], + "extra_cache_tags": "qaic-software-git-repo,qaic-software,qaic,software,kit" }, { "tags": "get,generic,sys-util,_libudev-dev" From 55f8d0502aab6a168a270b2c1153665ccd2522da Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 17:39:07 +0530 Subject: [PATCH 14/17] dont force llvm-prebuilt by default --- cm-mlops/script/get-llvm/_cm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cm-mlops/script/get-llvm/_cm.json b/cm-mlops/script/get-llvm/_cm.json index f75a38fdef..cefda36c15 100644 --- a/cm-mlops/script/get-llvm/_cm.json +++ b/cm-mlops/script/get-llvm/_cm.json @@ -35,7 +35,8 @@ ] }, "reuse_version": true, - "tags": "install,llvm,prebuilt" + "tags": "install,llvm", + "names": "llvm-install" } ], "sort": 100, From b33aac1406ab6d1f7beec9be54543faed35a35e6 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 18:44:59 +0530 Subject: [PATCH 15/17] Support external repos for qaic-software-kit --- .../script/get-qaic-software-kit/_cm.json | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cm-mlops/script/get-qaic-software-kit/_cm.json b/cm-mlops/script/get-qaic-software-kit/_cm.json index beffb2ecc9..2bd1d67d23 100644 --- a/cm-mlops/script/get-qaic-software-kit/_cm.json +++ b/cm-mlops/script/get-qaic-software-kit/_cm.json @@ -6,7 +6,11 @@ "category": "AI/ML frameworks", "deps": [ { - "tags": "get,git,repo,_repo.https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100", + "tags": "get,git,repo", + "update_tags_from_env_with_prefix": { + "_repo.": [ "CM_GIT_URL" ], + "_branch.": [ "CM_GIT_CHECKOUT" ] + }, "names": [ "qaic-software-git-repo" ], @@ -54,6 +58,25 @@ "qaic-software-kit" ], "uid": "3344655922694bbb", - "variations": {}, + "variations": { + "repo.quic": { + "group": "repo-source", + "default": true, + "env": { + "CM_GIT_URL": "https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100" + } + }, + "repo.#": { + "group": "repo-source", + "env": { + "CM_GIT_URL": "#" + } + }, + "branch.#": { + "env": { + "CM_GIT_CHECKOUT": "#" + } + } + }, "versions": {} } From 541a5bd8b381b0a88e0b1522fd7b830a4f7f3523 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 19:02:14 +0530 Subject: [PATCH 16/17] Support generic repos for qualcomm-cloud-ai-100-cc --- .../_cm.json | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json b/cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json index 9a5e990cfc..4db7d17b7c 100644 --- a/cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json +++ b/cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json @@ -9,7 +9,12 @@ "names": [ "qaic-software-git-repo" ], - "tags": "get,git,repo,_repo.https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100-cc" + "update_tags_from_env_with_prefix": { + "_repo.": [ "CM_GIT_URL" ], + "_branch.": [ "CM_GIT_CHECKOUT" ] + }, + "tags": "get,git,repo,_repo.https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100-cc", + "extra_cache_tags": "compute-sdk,qaic,from.src" }, { "names": [ @@ -48,6 +53,7 @@ "tags": [ "get", "qaic", + "from.src", "software", "compute", "compute-sdk", @@ -74,6 +80,24 @@ "CM_QAIC_COMPUTE_SDK_INSTALL_MODE": "release-assert" }, "group": "installation-mode" + }, + "repo.quic": { + "group": "repo-source", + "default": true, + "env": { + "CM_GIT_URL": "https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100-cc" + } + }, + "repo.#": { + "group": "repo-source", + "env": { + "CM_GIT_URL": "#" + } + }, + "branch.#": { + "env": { + "CM_GIT_CHECKOUT": "#" + } } }, "versions": {} From 9c0db07d3b1fa4c097950635df9f36c7ace9ac4e Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Sat, 10 Feb 2024 19:31:35 +0530 Subject: [PATCH 17/17] Add ninja dependency for qaic-compute-sdk --- cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json b/cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json index 4db7d17b7c..03638840f2 100644 --- a/cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json +++ b/cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json @@ -38,6 +38,9 @@ }, { "tags": "get,google,test" + }, + { + "tags": "get,generic-sys-util,_ninja-build" } ], "input_description": {},