Merge from CTuning (#1100)

cm-mlops/cfg/benchmark-run-loadgen-cpp-dev/any-model-offline.yaml

@@ -1,4 +1,4 @@
-uid: 4df38ed8dd804678
+uid: 125abafe58dc4473
 
 name: "Any model - offline"
 

cm-mlops/cfg/benchmark-run-loadgen-python-dev/any-model-offline.yaml

@@ -1,4 +1,4 @@
-uid: 4df38ed8dd804678
+uid: db45dcd686854602
 
 name: "Any model - offline"
 

cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/bert-reference-python-onnx-offline.yaml

@@ -1,4 +1,4 @@
-uid: 4df38ed8dd804678
+uid: 9eee8cb06621413a
 
 name: "BERT Reference Python ONNX Offline"
 

cm-mlops/cfg/benchmark-run-mlperf-inference-v3.1/gptj-reference-python-torch-offline.yaml

@@ -1,4 +1,4 @@
-uid: 4df38ed8dd804678
+uid: 53e4028a3b31400d
 
 name: "GPT-J Reference Python Torch Offline"
 

cm-mlops/script/build-dockerfile/customize.py

@@ -111,6 +111,10 @@ def preprocess(i):
     f.write('RUN '+ get_value(env, config, 'package-manager-get-cmd') + " " + " ".join(get_value(env, config,
         'packages')) + EOL)
 
+    if env['CM_DOCKER_OS'] == "ubuntu":
+        if int(env['CM_DOCKER_OS_VERSION'].split('.')[0]) >= 23:
+            if "--break-system-packages" not in env.get('CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS', ''):
+              env['CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS'] = " --break-system-packages"
     pip_extra_flags = env.get('CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS', '')
 
     f.write(EOL+'# Install python packages' + EOL)

cm-mlops/script/generate-mlperf-inference-user-conf/customize.py

@@ -55,7 +55,7 @@ def preprocess(i):
     RUN_CMD = ""
     state['RUN'] = {}
     test_list = ["TEST01", "TEST04", "TEST05"]
-    if env['CM_MODEL'] in ["rnnt", "bert-99", "bert-99.9", "dlrm-99", "dlrm-99.9", "3d-unet-99", "3d-unet-99.9"]:
+    if env['CM_MODEL'] in ["rnnt", "bert-99", "bert-99.9", "dlrm-v2-99", "dlrm-v2-99.9", "3d-unet-99", "3d-unet-99.9"]:
         test_list.remove("TEST04")
     if "gpt-" in env['CM_MODEL']:
         test_list.remove("TEST05")
@@ -93,7 +93,7 @@ def preprocess(i):
     if 'bert' in ml_model_name:
         ml_model_name = "bert"
     if 'dlrm' in ml_model_name:
-        ml_model_name = "dlrm"
+        ml_model_name = "dlrm-v2"
     if '3d-unet' in ml_model_name:
         ml_model_name = "3d-unet"
     if 'gptj' in ml_model_name:

cm-mlops/script/get-google-test/_cm.json

@@ -6,14 +6,20 @@
   "category": "Detection or installation of tools and artifacts",
   "deps": [
     {
-      "tags": "get,cmake"
+      "tags": "get,cmake",
+      "names": [
+        "cmake"
+      ]
     },
     {
-      "tags": "get,gcc"
+      "tags": "get,compiler",
+      "names": [
+        "compiler"
+      ]
     }
   ],
   "input_description": {},
-  "default_version": "1.13.0",
+  "default_version": "1.14.0",
   "input_mapping": {},
   "new_env_keys": [
     "CM_GOOGLE_TEST_SRC_PATH",
@@ -29,7 +35,8 @@
       "force_env_keys": [
         "CM_GIT_*"
       ],
-      "tags": "get,git,repo,_repo.https://github.com/google/googletest.git"
+      "tags": "get,git,repo,_repo.https://github.com/google/googletest.git",
+      "extra_cache_tags": "google-test,gtest"
     }
   ],
   "tags": [
@@ -41,6 +48,5 @@
     "google"
   ],
   "uid": "02945138a5614253",
-  "variations": {},
   "versions": {}
 }

cm-mlops/script/get-google-test/run.sh

@@ -1,8 +1,16 @@
 #!/bin/bash
+function cmake() {
+${CM_CMAKE_BIN_WITH_PATH} $@
+}
+
+export CC=${CM_C_COMPILER_WITH_PATH}
+export CXX=${CM_CXX_COMPILER_WITH_PATH}
+
 CUR=$PWD
 mkdir -p install
 INSTALL_DIR=$CUR/install
 cd ${CM_GIT_REPO_CHECKOUT_PATH}
+
 mkdir build
 cd build
 export MAKEFLAGS=-j${CM_MAKE_CORES}

cm-mlops/script/get-llvm/_cm.json

@@ -35,7 +35,8 @@
         ]
       },
       "reuse_version": true,
-      "tags": "install,llvm"
+      "tags": "install,llvm",
+      "names": "llvm-install"
     }
   ],
   "sort": 100,

cm-mlops/script/get-qaic-software-kit/_cm.json

@@ -6,10 +6,24 @@
   "category": "AI/ML frameworks",
   "deps": [
     {
-      "tags": "get,git,repo,_repo.https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100",
+      "tags": "get,git,repo",
+      "update_tags_from_env_with_prefix": {
+        "_repo.": [ "CM_GIT_URL" ],
+        "_branch.": [ "CM_GIT_CHECKOUT" ]
+      },
       "names": [
         "qaic-software-git-repo"
-      ]
+      ],
+      "extra_cache_tags": "qaic-software-git-repo,qaic-software,qaic,software,kit"
+    },
+    {
+      "tags": "get,generic,sys-util,_libudev-dev"
+    },
+    {
+      "tags": "get,generic,sys-util,_libpci-dev"
+    },
+    {
+      "tags": "get,google,test"
     },
     {
       "tags": "get,cmake",
@@ -23,15 +37,6 @@
       "names": [
         "compiler"
       ]
-    },
-    {
-      "tags": "get,generic,sys-util,_libudev-dev"
-    },
-    {
-      "tags": "get,generic,sys-util,_libpci-dev"
-    },
-    {
-      "tags": "get,google,test"
     }
   ],
   "input_description": {},
@@ -53,6 +58,25 @@
     "qaic-software-kit"
   ],
   "uid": "3344655922694bbb",
-  "variations": {},
+  "variations": {
+    "repo.quic": {
+      "group": "repo-source",
+      "default": true,
+      "env": {
+        "CM_GIT_URL": "https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100"
+      }
+    },
+    "repo.#": {
+      "group": "repo-source",
+      "env": {
+        "CM_GIT_URL": "#"
+      }
+    },
+    "branch.#": {
+      "env": {
+        "CM_GIT_CHECKOUT": "#"
+      }
+    }
+  },
   "versions": {}
 }

cm-mlops/script/get-qaic-software-kit/customize.py

@@ -15,6 +15,37 @@ def preprocess(i):
 
     quiet = (env.get('CM_QUIET', False) == 'yes')
 
+    if env.get('+ CXXFLAGS', []) == []:
+        env['+ CXXFLAGS'] = []
+    if env.get('+ CFLAGS', []) == []:
+        env['+ CFLAGS'] = []
+
+    if env.get('CM_LLVM_CLANG_VERSION', '') != '':
+        clang_version_split = env['CM_LLVM_CLANG_VERSION'].split(".")
+        clang_major_version = int(clang_version_split[0])
+
+        if clang_major_version >= 17:
+            env['+ CFLAGS'].append("-Wno-error=c2x-extensions")
+
+        if clang_major_version >= 16:
+            env['+ CFLAGS'].append("-Wno-error=unused-but-set-variable")
+            env['+ CXXFLAGS'].append("-Wno-error=unused-but-set-variable")
+
+        if clang_major_version >= 13:
+            env['+ CFLAGS'].append("-Wno-error=unused-const-variable")
+            env['+ CFLAGS'].append("-Wno-error=unused-but-set-variable")
+            env['+ CFLAGS'].append("-Wno-error=strict-prototypes")
+            env['+ CFLAGS'].append("-Wno-error=unused-variable")
+            env['+ CXXFLAGS'].append("-Wno-error=unused-const-variable")
+            env['+ CXXFLAGS'].append("-Wno-error=unused-variable")
+            env['+ CXXFLAGS'].append("-Wno-error=unused-private-field")
+            env['+ CXXFLAGS'].append("-Wno-error=unused-result")
+            env['+ CXXFLAGS'].append("-Wno-error=string-concatenation")
+            env['+ CXXFLAGS'].append("-Wno-error=infinite-recursion")
+
+        if clang_major_version == 12:
+            env['+ CXXFLAGS'].append("-Wno-error=unknown-warning-option")
+
     return {'return':0}
 
 def postprocess(i):

cm-mlops/script/get-qaic-software-kit/run.sh

@@ -4,6 +4,9 @@ function cmake() {
 ${CM_CMAKE_BIN_WITH_PATH} $@
 }
 
+export CC=${CM_C_COMPILER_WITH_PATH}
+export CXX=${CM_CXX_COMPILER_WITH_PATH}
+
 export -f cmake
 cd ${CM_QAIC_SOFTWARE_KIT_PATH}
 rm -rf build

cm-mlops/script/gui/playground_howtorun.py

@@ -54,6 +54,7 @@ def page(st, params, action = ''):
         initialized = True
 
     ii = {'streamlit_module': st,
+          'params': params,
           'meta': external_module_meta,
           'skip_title': True}
 

cm-mlops/script/install-llvm-prebuilt/customize.py

@@ -96,8 +96,8 @@ def preprocess(i):
 
                   elif need_version == '12.0.1':
                       default_os = '16.04'
-                      if host_os_version.startswith('18') or host_os_version.startswith('20'):
-                         default_os = '18.04'
+                      #if host_os_version.startswith('18') or host_os_version.startswith('20'):
+                      #   default_os = '18.04'
 
                   elif need_version == '13.0.0':
                       default_os = '16.04'

cm-mlops/script/install-qaic-compute-sdk-from-src/_cm.json

@@ -9,7 +9,12 @@
       "names": [
         "qaic-software-git-repo"
       ],
-      "tags": "get,git,repo,_repo.https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100-cc"
+       "update_tags_from_env_with_prefix": {
+        "_repo.": [ "CM_GIT_URL" ],
+        "_branch.": [ "CM_GIT_CHECKOUT" ]
+      },
+      "tags": "get,git,repo,_repo.https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100-cc",
+      "extra_cache_tags": "compute-sdk,qaic,from.src"
     },
     {
       "names": [
@@ -33,6 +38,9 @@
     },
     {
       "tags": "get,google,test"
+    },
+    {
+      "tags": "get,generic-sys-util,_ninja-build"
     }
   ],
   "input_description": {},
@@ -48,6 +56,7 @@
   "tags": [
     "get",
     "qaic",
+    "from.src",
     "software",
     "compute",
     "compute-sdk",
@@ -74,6 +83,24 @@
         "CM_QAIC_COMPUTE_SDK_INSTALL_MODE": "release-assert"
       },
       "group": "installation-mode"
+    },
+    "repo.quic": {
+      "group": "repo-source",
+      "default": true,
+      "env": {
+        "CM_GIT_URL": "https://github.com/quic/software-kit-for-qualcomm-cloud-ai-100-cc"
+      }
+    },
+    "repo.#": {
+      "group": "repo-source",
+      "env": {
+        "CM_GIT_URL": "#"
+      }
+    },
+    "branch.#": {
+      "env": {
+        "CM_GIT_CHECKOUT": "#"
+      }
     }
   },
   "versions": {}