Merge from CTuning (Conda support in CM) (#1051)

cm-mlops/automation/script/module.py

@@ -2363,6 +2363,8 @@ def _get_variation_groups(script, variations):
 
         for k in variations:
             variation = variations[k]
+            if not variation:
+                continue
             if 'group' in variation:
                 if variation['group'] not in groups:
                     groups[variation['group']] = {}
@@ -4184,7 +4186,7 @@ def append_deps(deps, new_deps):
                 dep = deps[i]
                 dep_names = dep.get('names',[])
                 if len(dep_names)>0:
-                    if set(new_dep_names) & set(dep_names):
+                    if set(new_dep_names) == set(dep_names):
                         deps[i] = new_dep
                         existing = True
                         break

cm-mlops/automation/utils/module.py

@@ -757,3 +757,26 @@ def list_files_recursively(self, i):
         print (s)
 
         return {'return':0}
+
+    ##############################################################################
+    def generate_secret(self, i):
+        """
+        Generate secret for web apps
+
+        Args:    
+
+        Returns:
+           (CM return dict):
+
+           secret (str): secret
+           
+           * return (int): return code == 0 if no error and >0 if error
+           * (error) (str): error string if return>0
+        """
+
+        import secrets
+        s = secrets.token_urlsafe(16)       
+
+        print (s)
+
+        return {'return':0, 'secret': s}

cm-mlops/script/app-loadgen-generic-python/_cm.yaml

@@ -32,6 +32,7 @@ input_mapping:
   interop: CM_MLPERF_INTEROP
   execmode: CM_MLPERF_EXEC_MODE
   modelpath: CM_ML_MODEL_FILE_WITH_PATH
+  samples: CM_MLPERF_LOADGEN_SAMPLES
 
 new_env_keys:
   - CM_MLPERF_*

cm-mlops/script/app-loadgen-generic-python/customize.py

@@ -6,27 +6,43 @@ def preprocess(i):
 
     os_info = i['os_info']
 
-    if os_info['platform'] == 'windows':
-        return {'return':1, 'error': 'Windows is not supported in this script yet'}
+#    if os_info['platform'] == 'windows':
+#        return {'return':1, 'error': 'Windows is not supported in this script yet'}
+
     env = i['env']
 
     if 'CM_ML_MODEL_FILE_WITH_PATH' not in env:
         return {'return': 1, 'error': 'Please select a variation specifying the model to run'}
 
-    env['CM_RUN_OPTS'] = ''
-    run_opts = ''
+    run_opts = env.get('CM_RUN_OPTS', '')
+
     if 'CM_MLPERF_RUNNER' in env:
         run_opts +=" -r "+env['CM_MLPERF_RUNNER']
+
     if 'CM_MLPERF_CONCURRENCY' in env:
         run_opts +=" --concurrency "+env['CM_MLPERF_CONCURRENCY']
+
     if 'CM_MLPERF_EXECUTION_PROVIDER' in env:
         run_opts +=" --ep "+env['CM_MLPERF_EXECUTION_PROVIDER']
+
     if 'CM_MLPERF_INTRAOP' in env:
         run_opts +=" --intraop "+env['CM_MLPERF_INTRAOP']
+
     if 'CM_MLPERF_INTEROP' in env:
         run_opts +=" --interop "+env['CM_MLPERF_INTEROP']
+
     if 'CM_MLPERF_EXECMODE' in env:
         run_opts +=" --execmode "+env['CM_MLPERF_EXECUTION_MODE']
+
+    if 'CM_MLPERF_LOADGEN_SAMPLES' in env:
+        run_opts +=" --samples "+env['CM_MLPERF_LOADGEN_SAMPLES']
+
+    env['CM_RUN_OPTS'] = run_opts
+
+    print ('')
+    print ('Assembled flags: {}'.format(run_opts))
+    print ('')    
+
     return {'return':0}
 
 def postprocess(i):

cm-mlops/script/app-loadgen-generic-python/run.bat

@@ -0,0 +1,4 @@
+rem native script
+
+%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\src\main.py %CM_RUN_OPTS% %CM_ML_MODEL_FILE_WITH_PATH%
+IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%

cm-mlops/script/app-loadgen-generic-python/src/main.py

@@ -22,7 +22,6 @@
 
 
 LOADGEN_EXPECTED_QPS = 50
-LOADGEN_SAMPLE_COUNT = 100
 LOADGEN_DURATION_SEC = 10
 
 
@@ -35,14 +34,18 @@ def main(
     execution_mode: str,
     intraop_threads: int,
     interop_threads: int,
+    samples: int
 ):
+
     model_factory = ORTModelFactory(
         model_path,
         execution_provider,
         execution_mode,
         interop_threads,
         intraop_threads,
     )
+
+
     model_dataset = ORTModelInputSampler(model_factory)
 
     runner: ModelRunner = None
@@ -71,7 +74,7 @@ def main(
     settings.scenario = mlperf_loadgen.TestScenario.Offline
     settings.mode = mlperf_loadgen.TestMode.PerformanceOnly
     settings.offline_expected_qps = LOADGEN_EXPECTED_QPS
-    settings.min_query_count = LOADGEN_SAMPLE_COUNT * 2
+    settings.min_query_count = samples * 2
     settings.min_duration_ms = LOADGEN_DURATION_SEC * 1000
     # Duration isn't enforced in offline mode
     # Instead, it is used to determine total sample count via
@@ -99,8 +102,8 @@ def main(
         harness = Harness(model_dataset, runner)
         try:
             query_sample_libary = mlperf_loadgen.ConstructQSL(
-                LOADGEN_SAMPLE_COUNT,  # Total sample count
-                LOADGEN_SAMPLE_COUNT,  # Num to load in RAM at a time
+                samples,  # Total sample count
+                samples,  # Num to load in RAM at a time
                 harness.load_query_samples,
                 harness.unload_query_samples,
             )
@@ -171,6 +174,12 @@ def main(
         choices=["sequential", "parallel"],
         default="sequential",
     )
+    parser.add_argument(
+        "--samples",
+        help="number of samples",
+        default=100,
+        type=int,
+    )
 
     args = parser.parse_args()
     main(
@@ -182,4 +191,5 @@ def main(
         args.execmode,
         args.intraop,
         args.interop,
+        args.samples
     )

cm-mlops/script/app-mlperf-inference/_cm.yaml

@@ -253,9 +253,28 @@ variations:
         update_tags_from_env:
             - CM_NVIDIA_HARNESS_GPU_VARIATION
 
+  intel:
+    group:
+      implementation
+    default_variations:
+      device: cpu
+      backend: pytorch
+    posthook_deps:
+      - names:
+         - intel
+         - intel-harness
+         - mlperf-inference-implementation
+        tags: reproduce,mlperf,inference,intel
+        skip_if_env:
+          CM_SKIP_RUN:
+            - yes
+
   kilt:
     group:
       implementation
+    default_variations:
+      device: qaic
+      backend: glow
     posthook_deps:
       - names:
          - kilt
@@ -478,7 +497,7 @@ variations:
         CM_DATASET_SQUAD_VAL_PATH: "on"
       tags: get,dataset,squad,language-processing
     - skip_if_env:
-        CM_ML_MODEL_BERT_VOCAB_FILE_WITH_PATH": "on"
+        CM_ML_MODEL_BERT_VOCAB_FILE_WITH_PATH: "on"
       tags: get,dataset-aux,squad-vocab
     post_deps:
     - enable_if_env:

cm-mlops/script/benchmark-object-detection-loadgen/run.bat

@@ -0,0 +1,7 @@
+rem native script
+
+echo "TBD"
+
+
+%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\\python\\main.py
+IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL%

cm-mlops/script/create-conda-env/_cm.json

@@ -0,0 +1,43 @@
+{
+  "alias": "create-conda-env",
+  "automation_alias": "script",
+  "automation_uid": "5b4e0237da074764",
+  "cache": true,
+  "category": "DevOps automation",
+  "clean_files": [],
+  "deps": [
+    {
+      "tags": "detect,os"
+    },
+    {
+      "names": [
+        "conda"
+      ],
+      "tags": "get,conda"
+    }
+  ],
+  "new_env_keys": [
+    "+PATH",
+    "+LD_LIBRARY_PATH",
+    "CM_CONDA_PREFIX",
+    "CONDA_PREFIX",
+    "CM_CONDA_BIN_PATH",
+    "CM_CONDA_LIB_PATH"
+  ],
+  "tags": [
+    "create",
+    "get",
+    "env",
+    "conda-env",
+    "conda-environment",
+    "create-conda-environment"
+  ],
+  "uid": "e39e0b04c86a40f2",
+  "variations": {
+    "name.#": {
+      "env": {
+        "CM_CONDA_ENV_NAME": "#"
+      }
+    }
+  }
+}

cm-mlops/script/create-conda-env/customize.py

@@ -0,0 +1,31 @@
+from cmind import utils
+import os
+
+def preprocess(i):
+
+    os_info = i['os_info']
+
+    env = i['env']
+    automation = i['automation']
+    run_script_input = i['run_script_input']
+
+    recursion_spaces = i['recursion_spaces']
+
+    if env.get('CM_CONDA_ENV_NAME', '') == '':
+        return {'return':1, 'error': 'Please use "_name.<conda env name>" variation'}
+
+    return {'return':0}
+
+def postprocess(i):
+    env = i['env']
+
+    conda_prefix = os.getcwd()
+    env['CONDA_PREFIX'] = conda_prefix
+    env['CM_CONDA_PREFIX'] = conda_prefix
+    env['CM_CONDA_BIN_PATH'] = os.path.join(conda_prefix, "bin")
+    env['CM_CONDA_LIB_PATH'] = os.path.join(conda_prefix, "lib")
+
+    env['+PATH'] = [ env['CM_CONDA_BIN_PATH'] ]
+    env['+LD_LIBRARY_PATH'] = [ env['CM_CONDA_LIB_PATH'] ]
+
+    return {'return':0}

cm-mlops/script/create-conda-env/run.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cmd="${CM_CONDA_BIN_WITH_PATH} create -p ${PWD}"
+echo "$cmd"
+eval "$cmd"
+test $? -eq 0 || exit $?
+

cm-mlops/script/get-conda/_cm.json

@@ -12,12 +12,24 @@
   ],
   "new_env_keys": [
     "+PATH",
-    "CM_CONDA_PREFIX"
+    "+LD_LIBRARY_PATH",
+    "CM_CONDA_PREFIX",
+    "CONDA_PREFIX",
+    "CM_CONDA_BIN_PATH",
+    "CM_CONDA_BIN_WITH_PATH",
+    "CM_CONDA_LIB_PATH"
   ],
   "tags": [
     "get",
     "conda",
     "get-conda"
   ],
-  "uid": "6600115f41324c7b"
+  "uid": "6600115f41324c7b",
+  "variations": {
+    "name.#": {
+      "env": {
+        "CM_CONDA_PREFIX_NAME": "#"
+      }
+    }
+  }
 }