Fix rclone config

cm-mlops/automation/script/module.py

@@ -2525,7 +2525,7 @@ def _call_run_deps(script, deps, local_env_keys, local_env_keys_from_meta, env,
     ##############################################################################
     def _run_deps(self, deps, clean_env_keys_deps, env, state, const, const_state, add_deps_recursive, recursion_spaces, 
                     remembered_selections, variation_tags_string='', from_cache=False, debug_script_tags='', 
-                    verbose=False, show_time=False, extra_recursion_spaces='  ', run_state={'deps':[], 'fake_deps':[]}):
+                  verbose=False, show_time=False, extra_recursion_spaces='  ', run_state={'deps':[], 'fake_deps':[], 'parent': None}):
         """
         Runs all the enabled dependencies and pass them env minus local env
         """

cm-mlops/automation/script/module_misc.py

@@ -1537,7 +1537,7 @@ def docker(i):
         return {'return':1, 'error':'no scripts were found'}
 
     env=i.get('env', {})
-    env['CM_RUN_STATE_DOCKER'] = True
+    env['CM_RUN_STATE_DOCKER'] = False
 
     docker_cache = i.get('docker_cache', "yes")
     if docker_cache in ["no", False, "False" ]:
@@ -1599,8 +1599,16 @@ def docker(i):
             update_path_for_docker('.', mounts, force_path_target=current_path_target)
 
 
-        _os = i.get('docker_os', meta.get('docker_os', 'ubuntu'))
-        version = i.get('docker_os_version', meta.get('docker_os_version', '22.04'))
+        _os = i.get('docker_os', docker_settings.get('docker_os', 'ubuntu'))
+        version = i.get('docker_os_version', docker_settings.get('docker_os_version', '22.04'))
+
+        deps = docker_settings.get('deps', [])
+        if deps:
+            # Todo: Support state, const and add_deps_recursive
+            script_automation = i['self_module']
+            r = script_automation._run_deps(deps, [], env, {}, {}, {}, {}, '',{})
+            if r['return'] > 0:
+                return r
 
         for key in docker_settings.get('mounts', []):
             mounts.append(key)
@@ -1722,6 +1730,7 @@ def docker(i):
         if r['return']>0: return r
 
         run_cmd  = r['run_cmd_string']
+        env['CM_RUN_STATE_DOCKER'] = True
 
         if docker_settings.get('mount_current_dir','')=='yes':
             run_cmd = 'cd '+current_path_target+' && '+run_cmd

cm-mlops/script/build-mlperf-inference-server-nvidia/_cm.yaml

@@ -228,6 +228,8 @@ docker:
     tensorrt_tar_file_path: CM_TENSORRT_TAR_FILE_PATH
     cuda_run_file_path: CUDA_RUN_FILE_LOCAL_PATH
     scratch_path: MLPERF_SCRATCH_PATH
+  deps:
+    - tags: get,mlperf,inference,nvidia,scratch,space
   mounts:
    - "${{ IMAGENET_PATH }}:/data/imagenet-val"
    - "${{ RESULTS_DIR }}:/home/cmuser/results_dir"

cm-mlops/script/download-file/customize.py

@@ -109,7 +109,7 @@ def preprocess(i):
         elif tool == "rclone":
             if env.get('CM_RCLONE_CONFIG_CMD', '') != '':
                 env['CM_DOWNLOAD_CONFIG_CMD'] = env['CM_RCLONE_CONFIG_CMD']
-            env['CM_DOWNLOAD_CMD'] = f"rclone copy {url} ./{env['CM_DOWNLOAD_FILENAME']} -P"
+            env['CM_DOWNLOAD_CMD'] = f"rclone copy {url} {os.path.join(os.getcwd(), env['CM_DOWNLOAD_FILENAME'])} -P"
 
         filename = env['CM_DOWNLOAD_FILENAME']
         env['CM_DOWNLOAD_DOWNLOADED_FILENAME'] = filename

cm-mlops/script/get-ml-model-gptj/_cm.json

@@ -19,6 +19,7 @@
   "prehook_deps": [
     {
       "env": {
+        "CM_DOWNLOAD_FINAL_ENV_NAME": "GPTJ_CHECKPOINT_PATH",
         "CM_EXTRACT_FINAL_ENV_NAME": "GPTJ_CHECKPOINT_PATH",
         "CM_EXTRACT_TO_FOLDER": "gpt-j"
       },
@@ -81,13 +82,14 @@
     "pytorch,fp32": {
       "env": {
         "CM_DOWNLOAD_EXTRA_OPTIONS": " --output-document checkpoint.zip",
-        "CM_DOWNLOAD_FILENAME": "checkpoint.zip",
         "CM_UNZIP": "yes",
         "CM_DOWNLOAD_CHECKSUM_NOT_USED": "e677e28aaf03da84584bb3073b7ee315",
         "CM_PACKAGE_URL": "https://cloud.mlcommons.org/index.php/s/QAZ2oM94MkFtbQx/download",
-        "CM_RCLONE_CONFIG": "rclone config create mlc-inference s3 provider=LyveCloud access_key_id=0LITLNQMHZALM5AK secret_access_key=YQKYTMBY23TMZHLOYFJKL5CHHS0CWYUC endpoint=s3.us-east-1.lyvecloud.seagate.com",
+        "CM_RCLONE_CONFIG_CMD": "rclone config create mlc-inference s3 provider=LyveCloud access_key_id=0LITLNQMHZALM5AK secret_access_key=YQKYTMBY23TMZHLOYFJKL5CHHS0CWYUC endpoint=s3.us-east-1.lyvecloud.seagate.com",
         "CM_RCLONE_URL": "mlc-inference:mlcommons-inference-wg-s3/gpt-j"
-      },
+      }
+    },
+    "pytorch,fp32,wget": {
       "add_deps_recursive": {
         "dae": {
           "tags": "_extract"
@@ -186,7 +188,8 @@
         }
       },
       "env": {
-        "CM_DOWNLOAD_URL": "<<<CM_PACKAGE_URL>>>"
+        "CM_DOWNLOAD_URL": "<<<CM_PACKAGE_URL>>>",
+        "CM_DOWNLOAD_FILENAME": "checkpoint.zip"
       }
     },
     "rclone": {
@@ -197,6 +200,7 @@
         }
       },
       "env": {
+        "CM_DOWNLOAD_FILENAME": "checkpoint",
         "CM_DOWNLOAD_URL": "<<<CM_RCLONE_URL>>>"
       }
     },

cm-mlops/script/get-ml-model-stable-diffusion/_cm.json

@@ -155,7 +155,7 @@
     "rclone": {
       "group": "download-tool",
       "env": {
-        "CM_RCLONE_CONFIG": "rclone config create mlc-inference s3 provider=LyveCloud access_key_id=0LITLNQMHZALM5AK secret_access_key=YQKYTMBY23TMZHLOYFJKL5CHHS0CWYUC endpoint=s3.us-east-1.lyvecloud.seagate.com",
+        "CM_RCLONE_CONFIG_CMD": "rclone config create mlc-inference s3 provider=LyveCloud access_key_id=0LITLNQMHZALM5AK secret_access_key=YQKYTMBY23TMZHLOYFJKL5CHHS0CWYUC endpoint=s3.us-east-1.lyvecloud.seagate.com",
         "CM_DOWNLOAD_TOOL": "rclone"
       },
       "adr": {

cm-mlops/script/get-rclone/run.sh

@@ -1,3 +1,7 @@
 #!/bin/bash
+if ! command -v rclone &> /dev/null
+then
+  exit 1
+fi
 rclone --version > tmp-ver.out
 test $? -eq 0 || exit 1